From bcb95b01124422ad818c70a7e3901b3e284f84ec Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Tue, 19 May 2026 14:06:33 +0200 Subject: [PATCH 01/43] Add basic ARM64 architecture support for NEON, SVE, and SVE2. Just a skeleton, no implementations yet. --- CMakeLists.txt | 10 +- external/simde | 2 +- include/pernix/arm64/neon/compression.h | 59 +++++++ include/pernix/arm64/neon/decompression.h | 59 +++++++ include/pernix/arm64/neon/packing.h | 11 ++ include/pernix/arm64/neon/unpacking.h | 20 +++ include/pernix/arm64/sve/compression.h | 59 +++++++ include/pernix/arm64/sve/decompression.h | 59 +++++++ include/pernix/arm64/sve/packing.h | 11 ++ include/pernix/arm64/sve/unpacking.h | 11 ++ include/pernix/arm64/sve2/compression.h | 59 +++++++ include/pernix/arm64/sve2/decompression.h | 59 +++++++ include/pernix/arm64/sve2/packing.h | 11 ++ include/pernix/arm64/sve2/unpacking.h | 11 ++ include/pernix/detection.h | 19 +- include/pernix/pernix.h | 167 +++++++++++++++++- include/pernix/simd_compat.h | 8 + src/CMakeLists.txt | 48 ++++- src/arm64/neon/compression.cpp | 21 +++ src/arm64/neon/decompression.cpp | 21 +++ src/arm64/sve/compression.cpp | 21 +++ src/arm64/sve/decompression.cpp | 21 +++ src/arm64/sve2/compression.cpp | 21 +++ src/arm64/sve2/decompression.cpp | 21 +++ src/pernix.cpp | 112 +++++++++++- tests/arm64/neon/.gitkeep | 0 tests/arm64/sve/.gitkeep | 0 tests/arm64/sve2/.gitkeep | 0 .../compression_tests.cpp} | 2 +- .../decompression_tests.cpp} | 2 +- .../edge_tests.cpp} | 0 tests/include/testset.h | 2 +- .../avx2/compression_tests.cpp} | 2 +- .../avx2/decompression_tests.cpp} | 2 +- .../avx512vbmi/compression_tests.cpp} | 0 .../avx512vbmi/decompression_tests.cpp} | 2 +- .../bmi2/compression_tests.cpp} | 2 +- .../bmi2/decompression_tests.cpp} | 2 +- 38 files changed, 916 insertions(+), 21 deletions(-) create mode 100644 include/pernix/arm64/neon/compression.h create mode 100644 include/pernix/arm64/neon/decompression.h create mode 100644 include/pernix/arm64/neon/packing.h create mode 100644 include/pernix/arm64/neon/unpacking.h create mode 100644 include/pernix/arm64/sve/compression.h create mode 100644 include/pernix/arm64/sve/decompression.h create mode 100644 include/pernix/arm64/sve/packing.h create mode 100644 include/pernix/arm64/sve/unpacking.h create mode 100644 include/pernix/arm64/sve2/compression.h create mode 100644 include/pernix/arm64/sve2/decompression.h create mode 100644 include/pernix/arm64/sve2/packing.h create mode 100644 include/pernix/arm64/sve2/unpacking.h create mode 100644 src/arm64/neon/compression.cpp create mode 100644 src/arm64/neon/decompression.cpp create mode 100644 src/arm64/sve/compression.cpp create mode 100644 src/arm64/sve/decompression.cpp create mode 100644 src/arm64/sve2/compression.cpp create mode 100644 src/arm64/sve2/decompression.cpp create mode 100644 tests/arm64/neon/.gitkeep create mode 100644 tests/arm64/sve/.gitkeep create mode 100644 tests/arm64/sve2/.gitkeep rename tests/{compression/fallback_compression_tests.cpp => fallback/compression_tests.cpp} (97%) rename tests/{decompression/fallback_decompression_tests.cpp => fallback/decompression_tests.cpp} (97%) rename tests/{fallback_edge_tests.cpp => fallback/edge_tests.cpp} (100%) rename tests/{compression/avx2_compression_tests.cpp => x86/avx2/compression_tests.cpp} (97%) rename tests/{decompression/avx2_decompression_tests.cpp => x86/avx2/decompression_tests.cpp} (97%) rename tests/{compression/avx512vbmi_compression_tests.cpp => x86/avx512vbmi/compression_tests.cpp} (100%) rename tests/{decompression/avx512vbmi_decompression_tests.cpp => x86/avx512vbmi/decompression_tests.cpp} (97%) rename tests/{compression/bmi2_compression_tests.cpp => x86/bmi2/compression_tests.cpp} (97%) rename tests/{decompression/bmi2_decompression_tests.cpp => x86/bmi2/decompression_tests.cpp} (97%) diff --git a/CMakeLists.txt b/CMakeLists.txt index edf6c78..3fa49be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,9 +13,17 @@ option(PERNIX_DISABLE_AVX2 "Disable AVX2 optimizations" off) option(PERNIX_DISABLE_AVX512 "Disable AVX512 optimizations" off) option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" off) +set(PERNIX_ARCH_BACKEND "AUTO" CACHE STRING "Pernix architecture backend (AUTO, FALLBACK, X86, ARM64_NEON, ARM64_SVE, ARM64_SVE2)") +set_property(CACHE PERNIX_ARCH_BACKEND PROPERTY STRINGS AUTO FALLBACK X86 ARM64_NEON ARM64_SVE ARM64_SVE2) option(PERNIX_ENABLE_FORTRAN_BINDINGS "Build Fortran bindings for pernix" off) +string(TOUPPER "${PERNIX_ARCH_BACKEND}" PERNIX_ARCH_BACKEND) +set(PERNIX_VALID_ARCH_BACKENDS AUTO FALLBACK X86 ARM64_NEON ARM64_SVE ARM64_SVE2) +if (NOT PERNIX_ARCH_BACKEND IN_LIST PERNIX_VALID_ARCH_BACKENDS) + message(FATAL_ERROR "Unsupported PERNIX_ARCH_BACKEND='${PERNIX_ARCH_BACKEND}'. Expected one of: ${PERNIX_VALID_ARCH_BACKENDS}") +endif () + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") if (PERNIX_USE_SIMDE) @@ -97,4 +105,4 @@ endif () if (PERNIX_ENABLE_TESTS) enable_testing() add_subdirectory(tests) -endif () \ No newline at end of file +endif () diff --git a/external/simde b/external/simde index 1747b24..1a1ca5e 160000 --- a/external/simde +++ b/external/simde @@ -1 +1 @@ -Subproject commit 1747b2482589fe894d49989159421da08c2a8bcd +Subproject commit 1a1ca5ee71518d8a115234dad1e2d871421953b7 diff --git a/include/pernix/arm64/neon/compression.h b/include/pernix/arm64/neon/compression.h new file mode 100644 index 0000000..65ea786 --- /dev/null +++ b/include/pernix/arm64/neon/compression.h @@ -0,0 +1,59 @@ +#ifndef PERNIX_ARM64_NEON_COMPRESSION_H +#define PERNIX_ARM64_NEON_COMPRESSION_H + +#include + +#include +#include + +namespace pernix { +namespace internal { +template +inline constexpr bool neon_compression_unimplemented_v = false; +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_compress_block(const float_t*, float_t, uint8_t*) { + static_assert(internal::neon_compression_unimplemented_v, "ARM64 NEON compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_compress_block(const double_t*, double_t, uint8_t*) { + static_assert(internal::neon_compression_unimplemented_v, "ARM64 NEON compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_compress_blocks(const float_t*, float_t, uint8_t*, uint32_t) { + static_assert(internal::neon_compression_unimplemented_v, "ARM64 NEON compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_compress_blocks(const double_t*, double_t, uint8_t*, uint32_t) { + static_assert(internal::neon_compression_unimplemented_v, "ARM64 NEON compression is not implemented yet"); + return -1; +} + +#ifdef __cplusplus +extern "C" { +#endif + +int neon_compress_block(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); +int neon_compress_block_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); +int neon_compress_blocks(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, + uint32_t blocks); +int neon_compress_blocks_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, + uint32_t blocks); + +#ifdef __cplusplus +} +#endif +} // namespace pernix + +#endif // PERNIX_ARM64_NEON_COMPRESSION_H diff --git a/include/pernix/arm64/neon/decompression.h b/include/pernix/arm64/neon/decompression.h new file mode 100644 index 0000000..0f8d79e --- /dev/null +++ b/include/pernix/arm64/neon/decompression.h @@ -0,0 +1,59 @@ +#ifndef PERNIX_ARM64_NEON_DECOMPRESSION_H +#define PERNIX_ARM64_NEON_DECOMPRESSION_H + +#include + +#include +#include + +namespace pernix { +namespace internal { +template +inline constexpr bool neon_decompression_unimplemented_v = false; +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_decompress_block(const uint8_t*, float_t, float_t*) { + static_assert(internal::neon_decompression_unimplemented_v, "ARM64 NEON decompression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_decompress_block(const uint8_t*, double_t, double_t*) { + static_assert(internal::neon_decompression_unimplemented_v, "ARM64 NEON decompression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_decompress_blocks(const uint8_t*, float_t, float_t*, uint32_t) { + static_assert(internal::neon_decompression_unimplemented_v, "ARM64 NEON decompression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_decompress_blocks(const uint8_t*, double_t, double_t*, uint32_t) { + static_assert(internal::neon_decompression_unimplemented_v, "ARM64 NEON decompression is not implemented yet"); + return -1; +} + +#ifdef __cplusplus +extern "C" { +#endif + +int neon_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); +int neon_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); +int neon_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, + uint32_t blocks); +int neon_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, + uint32_t blocks); + +#ifdef __cplusplus +} +#endif +} // namespace pernix + +#endif // PERNIX_ARM64_NEON_DECOMPRESSION_H diff --git a/include/pernix/arm64/neon/packing.h b/include/pernix/arm64/neon/packing.h new file mode 100644 index 0000000..c1c8119 --- /dev/null +++ b/include/pernix/arm64/neon/packing.h @@ -0,0 +1,11 @@ +#ifndef PERNIX_ARM64_NEON_PACKING_H +#define PERNIX_ARM64_NEON_PACKING_H + +#include + +namespace pernix::arm64::neon::internal { +template +inline constexpr bool packing_unimplemented_v = false; +} // namespace pernix::arm64::neon::internal + +#endif // PERNIX_ARM64_NEON_PACKING_H diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h new file mode 100644 index 0000000..32dc5de --- /dev/null +++ b/include/pernix/arm64/neon/unpacking.h @@ -0,0 +1,20 @@ +#ifndef PERNIX_ARM64_NEON_UNPACKING_H +#define PERNIX_ARM64_NEON_UNPACKING_H + +#include + +namespace pernix::arm64::neon::internal { + +template +inline constexpr bool unpacking_unimplemented_v = false; + +namespace b64 { + +} // namespace b64 + + +} // namespace pernix::arm64::neon::internal + + + +#endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/arm64/sve/compression.h b/include/pernix/arm64/sve/compression.h new file mode 100644 index 0000000..f8abfed --- /dev/null +++ b/include/pernix/arm64/sve/compression.h @@ -0,0 +1,59 @@ +#ifndef PERNIX_ARM64_SVE_COMPRESSION_H +#define PERNIX_ARM64_SVE_COMPRESSION_H + +#include + +#include +#include + +namespace pernix { +namespace internal { +template +inline constexpr bool sve_compression_unimplemented_v = false; +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve_compress_block(const float_t*, float_t, uint8_t*) { + static_assert(internal::sve_compression_unimplemented_v, "ARM64 SVE compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve_compress_block(const double_t*, double_t, uint8_t*) { + static_assert(internal::sve_compression_unimplemented_v, "ARM64 SVE compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve_compress_blocks(const float_t*, float_t, uint8_t*, uint32_t) { + static_assert(internal::sve_compression_unimplemented_v, "ARM64 SVE compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve_compress_blocks(const double_t*, double_t, uint8_t*, uint32_t) { + static_assert(internal::sve_compression_unimplemented_v, "ARM64 SVE compression is not implemented yet"); + return -1; +} + +#ifdef __cplusplus +extern "C" { +#endif + +int sve_compress_block(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); +int sve_compress_block_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); +int sve_compress_blocks(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, + uint32_t blocks); +int sve_compress_blocks_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, + uint32_t blocks); + +#ifdef __cplusplus +} +#endif +} // namespace pernix + +#endif // PERNIX_ARM64_SVE_COMPRESSION_H diff --git a/include/pernix/arm64/sve/decompression.h b/include/pernix/arm64/sve/decompression.h new file mode 100644 index 0000000..784c0c8 --- /dev/null +++ b/include/pernix/arm64/sve/decompression.h @@ -0,0 +1,59 @@ +#ifndef PERNIX_ARM64_SVE_DECOMPRESSION_H +#define PERNIX_ARM64_SVE_DECOMPRESSION_H + +#include + +#include +#include + +namespace pernix { +namespace internal { +template +inline constexpr bool sve_decompression_unimplemented_v = false; +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve_decompress_block(const uint8_t*, float_t, float_t*) { + static_assert(internal::sve_decompression_unimplemented_v, "ARM64 SVE decompression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve_decompress_block(const uint8_t*, double_t, double_t*) { + static_assert(internal::sve_decompression_unimplemented_v, "ARM64 SVE decompression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve_decompress_blocks(const uint8_t*, float_t, float_t*, uint32_t) { + static_assert(internal::sve_decompression_unimplemented_v, "ARM64 SVE decompression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve_decompress_blocks(const uint8_t*, double_t, double_t*, uint32_t) { + static_assert(internal::sve_decompression_unimplemented_v, "ARM64 SVE decompression is not implemented yet"); + return -1; +} + +#ifdef __cplusplus +extern "C" { +#endif + +int sve_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); +int sve_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); +int sve_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, + uint32_t blocks); +int sve_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, + uint32_t blocks); + +#ifdef __cplusplus +} +#endif +} // namespace pernix + +#endif // PERNIX_ARM64_SVE_DECOMPRESSION_H diff --git a/include/pernix/arm64/sve/packing.h b/include/pernix/arm64/sve/packing.h new file mode 100644 index 0000000..fce21ca --- /dev/null +++ b/include/pernix/arm64/sve/packing.h @@ -0,0 +1,11 @@ +#ifndef PERNIX_ARM64_SVE_PACKING_H +#define PERNIX_ARM64_SVE_PACKING_H + +#include + +namespace pernix::arm64::sve::internal { +template +inline constexpr bool packing_unimplemented_v = false; +} // namespace pernix::arm64::sve::internal + +#endif // PERNIX_ARM64_SVE_PACKING_H diff --git a/include/pernix/arm64/sve/unpacking.h b/include/pernix/arm64/sve/unpacking.h new file mode 100644 index 0000000..3de49ca --- /dev/null +++ b/include/pernix/arm64/sve/unpacking.h @@ -0,0 +1,11 @@ +#ifndef PERNIX_ARM64_SVE_UNPACKING_H +#define PERNIX_ARM64_SVE_UNPACKING_H + +#include + +namespace pernix::arm64::sve::internal { +template +inline constexpr bool unpacking_unimplemented_v = false; +} // namespace pernix::arm64::sve::internal + +#endif // PERNIX_ARM64_SVE_UNPACKING_H diff --git a/include/pernix/arm64/sve2/compression.h b/include/pernix/arm64/sve2/compression.h new file mode 100644 index 0000000..4e4627d --- /dev/null +++ b/include/pernix/arm64/sve2/compression.h @@ -0,0 +1,59 @@ +#ifndef PERNIX_ARM64_SVE2_COMPRESSION_H +#define PERNIX_ARM64_SVE2_COMPRESSION_H + +#include + +#include +#include + +namespace pernix { +namespace internal { +template +inline constexpr bool sve2_compression_unimplemented_v = false; +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_block(const float_t*, float_t, uint8_t*) { + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_block(const double_t*, double_t, uint8_t*) { + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_blocks(const float_t*, float_t, uint8_t*, uint32_t) { + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_blocks(const double_t*, double_t, uint8_t*, uint32_t) { + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +#ifdef __cplusplus +extern "C" { +#endif + +int sve2_compress_block(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); +int sve2_compress_block_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); +int sve2_compress_blocks(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, + uint32_t blocks); +int sve2_compress_blocks_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, + uint32_t blocks); + +#ifdef __cplusplus +} +#endif +} // namespace pernix + +#endif // PERNIX_ARM64_SVE2_COMPRESSION_H diff --git a/include/pernix/arm64/sve2/decompression.h b/include/pernix/arm64/sve2/decompression.h new file mode 100644 index 0000000..c27f08a --- /dev/null +++ b/include/pernix/arm64/sve2/decompression.h @@ -0,0 +1,59 @@ +#ifndef PERNIX_ARM64_SVE2_DECOMPRESSION_H +#define PERNIX_ARM64_SVE2_DECOMPRESSION_H + +#include + +#include +#include + +namespace pernix { +namespace internal { +template +inline constexpr bool sve2_decompression_unimplemented_v = false; +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_block(const uint8_t*, float_t, float_t*) { + static_assert(internal::sve2_decompression_unimplemented_v, "ARM64 SVE2 decompression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_block(const uint8_t*, double_t, double_t*) { + static_assert(internal::sve2_decompression_unimplemented_v, "ARM64 SVE2 decompression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_blocks(const uint8_t*, float_t, float_t*, uint32_t) { + static_assert(internal::sve2_decompression_unimplemented_v, "ARM64 SVE2 decompression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_blocks(const uint8_t*, double_t, double_t*, uint32_t) { + static_assert(internal::sve2_decompression_unimplemented_v, "ARM64 SVE2 decompression is not implemented yet"); + return -1; +} + +#ifdef __cplusplus +extern "C" { +#endif + +int sve2_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); +int sve2_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); +int sve2_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, + uint32_t blocks); +int sve2_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, + uint32_t blocks); + +#ifdef __cplusplus +} +#endif +} // namespace pernix + +#endif // PERNIX_ARM64_SVE2_DECOMPRESSION_H diff --git a/include/pernix/arm64/sve2/packing.h b/include/pernix/arm64/sve2/packing.h new file mode 100644 index 0000000..789b4d7 --- /dev/null +++ b/include/pernix/arm64/sve2/packing.h @@ -0,0 +1,11 @@ +#ifndef PERNIX_ARM64_SVE2_PACKING_H +#define PERNIX_ARM64_SVE2_PACKING_H + +#include + +namespace pernix::arm64::sve2::internal { +template +inline constexpr bool packing_unimplemented_v = false; +} // namespace pernix::arm64::sve2::internal + +#endif // PERNIX_ARM64_SVE2_PACKING_H diff --git a/include/pernix/arm64/sve2/unpacking.h b/include/pernix/arm64/sve2/unpacking.h new file mode 100644 index 0000000..d654b5e --- /dev/null +++ b/include/pernix/arm64/sve2/unpacking.h @@ -0,0 +1,11 @@ +#ifndef PERNIX_ARM64_SVE2_UNPACKING_H +#define PERNIX_ARM64_SVE2_UNPACKING_H + +#include + +namespace pernix::arm64::sve2::internal { +template +inline constexpr bool unpacking_unimplemented_v = false; +} // namespace pernix::arm64::sve2::internal + +#endif // PERNIX_ARM64_SVE2_UNPACKING_H diff --git a/include/pernix/detection.h b/include/pernix/detection.h index edecb6c..fa9cd44 100644 --- a/include/pernix/detection.h +++ b/include/pernix/detection.h @@ -10,6 +10,19 @@ #define PERNIX_MACHINE_ID_V4 3 #define PERNIX_MACHINE_ID_V4_VBMI 4 +#if defined(PERNIX_BACKEND_ARM64_NEON) +#define PERNIX_ARM64_NEON_ENABLED +#endif + +#if defined(PERNIX_BACKEND_ARM64_SVE) +#define PERNIX_ARM64_SVE_ENABLED +#endif + +#if defined(PERNIX_BACKEND_ARM64_SVE2) +#define PERNIX_ARM64_SVE2_ENABLED +#endif + +#if defined(PERNIX_BACKEND_X86) // Map the compiler's enabled ISA set to the highest supported Pernix target level. #if (__SSE3__ && __SSE4_1__ && __SSE4_2__) #if (__AVX__ && __AVX2__ && __FMA__ && __BMI__ && __BMI2__) @@ -32,6 +45,10 @@ #define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_GENERIC #endif +#else +#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_GENERIC +#endif + // Feature-selection macros consumed by the public headers. #if (PERNIX_MACHINE_ID >= PERNIX_MACHINE_ID_V2) #define PERNIX_SSE_ENABLED @@ -47,7 +64,7 @@ #define PERNIX_AVX512_VBMI_ENABLED #endif -#ifdef PERNIX_USE_SIMDE +#if defined(PERNIX_USE_SIMDE) && defined(PERNIX_BACKEND_X86) #define PERNIX_SSE_ENABLED #define PERNIX_AVX2_ENABLED #define PERNIX_BMI2_ENABLED diff --git a/include/pernix/pernix.h b/include/pernix/pernix.h index 55133bb..4998a60 100644 --- a/include/pernix/pernix.h +++ b/include/pernix/pernix.h @@ -5,7 +5,7 @@ // Include architecture-specific headers based on detected capabilities // AVX2 -#ifdef PERNIX_AVX2_ENABLED +#if defined(PERNIX_BACKEND_X86) && defined(PERNIX_AVX2_ENABLED) #include #include @@ -21,7 +21,22 @@ #include #endif // PERNIX_AVX512_VBMI_ENABLED -#endif // PERNIX_AVX2_ENABLED +#endif // PERNIX_BACKEND_X86 && PERNIX_AVX2_ENABLED + +#ifdef PERNIX_BACKEND_ARM64_NEON +#include +#include +#endif + +#ifdef PERNIX_BACKEND_ARM64_SVE +#include +#include +#endif + +#ifdef PERNIX_BACKEND_ARM64_SVE2 +#include +#include +#endif // Fallback (non-SIMD) implementations #include @@ -167,7 +182,7 @@ template int decompress_blocks(const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, uint32_t blocks); // Use the best available implementation based on detected CPU features at compile time. -#ifdef PERNIX_AVX2_ENABLED +#if defined(PERNIX_BACKEND_X86) && defined(PERNIX_AVX2_ENABLED) #ifdef PERNIX_AVX512_VBMI_ENABLED template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) @@ -265,6 +280,150 @@ int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, d return mm256_decompress_blocks_avx2(input, scale, output, blocks); } #endif +#elif defined(PERNIX_BACKEND_ARM64_NEON) +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { + return neon_compress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { + return neon_compress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { + return neon_compress_blocks(input, scale, output, blocks); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { + return neon_compress_blocks(input, scale, output, blocks); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + return neon_decompress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + return neon_decompress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { + return neon_decompress_blocks(input, scale, output, blocks); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { + return neon_decompress_blocks(input, scale, output, blocks); +} +#elif defined(PERNIX_BACKEND_ARM64_SVE) +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { + return sve_compress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { + return sve_compress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { + return sve_compress_blocks(input, scale, output, blocks); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { + return sve_compress_blocks(input, scale, output, blocks); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + return sve_decompress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + return sve_decompress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { + return sve_decompress_blocks(input, scale, output, blocks); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { + return sve_decompress_blocks(input, scale, output, blocks); +} +#elif defined(PERNIX_BACKEND_ARM64_SVE2) +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { + return sve2_compress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { + return sve2_compress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { + return sve2_compress_blocks(input, scale, output, blocks); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { + return sve2_compress_blocks(input, scale, output, blocks); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + return sve2_decompress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + return sve2_decompress_block(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { + return sve2_decompress_blocks(input, scale, output, blocks); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { + return sve2_decompress_blocks(input, scale, output, blocks); +} #else template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) @@ -420,4 +579,4 @@ int decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, } // namespace pernix #endif -#endif // PERNIX_H \ No newline at end of file +#endif // PERNIX_H diff --git a/include/pernix/simd_compat.h b/include/pernix/simd_compat.h index c95c1ee..8a66ee7 100644 --- a/include/pernix/simd_compat.h +++ b/include/pernix/simd_compat.h @@ -8,9 +8,15 @@ #define SIMDE_ENABLE_NATIVE_ALIASES #undef SIMDE_X86_AVX512FP16_NATIVE // #define SIMDE_NO_NATIVE +#if defined(PERNIX_BACKEND_X86) #include #include #include +#elif defined(PERNIX_BACKEND_ARM64_NEON) +#include +#elif defined(PERNIX_BACKEND_ARM64_SVE) || defined(PERNIX_BACKEND_ARM64_SVE2) +#include +#endif // #ifndef __mmask8 // typedef uint8_t __mmask8; @@ -27,6 +33,8 @@ #elif defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) #include +#elif defined(__aarch64__) +#include #endif #ifndef __always_inline diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b7e8f23..f0c7e3e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,18 +9,46 @@ file(GLOB_RECURSE set(PERNIX_SOURCES ${PERNIX_COMMON_SOURCES}) -set(PERNIX_TARGET_IS_X86 OFF) -if (PERNIX_USE_SIMDE OR CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|i[3-6]86|i686)$") - set(PERNIX_TARGET_IS_X86 ON) +set(PERNIX_SELECTED_ARCH_BACKEND "${PERNIX_ARCH_BACKEND}") +if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "AUTO") + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|i[3-6]86|i686)$") + set(PERNIX_SELECTED_ARCH_BACKEND "X86") + elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") + set(PERNIX_SELECTED_ARCH_BACKEND "ARM64_NEON") + else () + set(PERNIX_SELECTED_ARCH_BACKEND "FALLBACK") + endif () endif () +message(STATUS "Pernix architecture backend: ${PERNIX_SELECTED_ARCH_BACKEND}") -if (PERNIX_TARGET_IS_X86) +if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "X86") file(GLOB_RECURSE PERNIX_X86_SOURCES ./x86/*.cpp ${PROJECT_SOURCE_DIR}/include/pernix/x86/*.h ) list(APPEND PERNIX_SOURCES ${PERNIX_X86_SOURCES}) +elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_NEON") + file(GLOB_RECURSE + PERNIX_ARM64_NEON_SOURCES + ./arm64/neon/*.cpp + ${PROJECT_SOURCE_DIR}/include/pernix/arm64/neon/*.h + ) + list(APPEND PERNIX_SOURCES ${PERNIX_ARM64_NEON_SOURCES}) +elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE") + file(GLOB_RECURSE + PERNIX_ARM64_SVE_SOURCES + ./arm64/sve/*.cpp + ${PROJECT_SOURCE_DIR}/include/pernix/arm64/sve/*.h + ) + list(APPEND PERNIX_SOURCES ${PERNIX_ARM64_SVE_SOURCES}) +elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE2") + file(GLOB_RECURSE + PERNIX_ARM64_SVE2_SOURCES + ./arm64/sve2/*.cpp + ${PROJECT_SOURCE_DIR}/include/pernix/arm64/sve2/*.h + ) + list(APPEND PERNIX_SOURCES ${PERNIX_ARM64_SVE2_SOURCES}) endif () add_library(pernix SHARED ${PERNIX_SOURCES}) @@ -37,6 +65,18 @@ if (PERNIX_USE_SIMDE) target_compile_definitions(pernix PUBLIC PERNIX_USE_SIMDE=1) endif () +target_compile_definitions(pernix PUBLIC "PERNIX_BACKEND_${PERNIX_SELECTED_ARCH_BACKEND}=1") + +if (PERNIX_DISABLE_BMI2) + target_compile_definitions(pernix PUBLIC PERNIX_DISABLE_BMI2=1) +endif () +if (PERNIX_DISABLE_AVX2) + target_compile_definitions(pernix PUBLIC PERNIX_DISABLE_AVX2=1) +endif () +if (PERNIX_DISABLE_AVX512) + target_compile_definitions(pernix PUBLIC PERNIX_DISABLE_AVX512=1) +endif () + set_target_properties(pernix PROPERTIES LINKER_LANGUAGE CXX) configure_file( diff --git a/src/arm64/neon/compression.cpp b/src/arm64/neon/compression.cpp new file mode 100644 index 0000000..5968f79 --- /dev/null +++ b/src/arm64/neon/compression.cpp @@ -0,0 +1,21 @@ +#include + +namespace pernix { +extern "C" { +int neon_compress_block(uint8_t, const float_t*, float_t, uint8_t*) { + return -1; +} + +int neon_compress_block_f64(uint8_t, const double_t*, double_t, uint8_t*) { + return -1; +} + +int neon_compress_blocks(uint8_t, const float_t*, float_t, uint8_t*, uint32_t) { + return -1; +} + +int neon_compress_blocks_f64(uint8_t, const double_t*, double_t, uint8_t*, uint32_t) { + return -1; +} +} +} // namespace pernix diff --git a/src/arm64/neon/decompression.cpp b/src/arm64/neon/decompression.cpp new file mode 100644 index 0000000..3cb7fc7 --- /dev/null +++ b/src/arm64/neon/decompression.cpp @@ -0,0 +1,21 @@ +#include + +namespace pernix { +extern "C" { +int neon_decompress_block(uint8_t, const uint8_t*, float_t, float_t*) { + return -1; +} + +int neon_decompress_block_f64(uint8_t, const uint8_t*, double_t, double_t*) { + return -1; +} + +int neon_decompress_blocks(uint8_t, const uint8_t*, float_t, float_t*, uint32_t) { + return -1; +} + +int neon_decompress_blocks_f64(uint8_t, const uint8_t*, double_t, double_t*, uint32_t) { + return -1; +} +} +} // namespace pernix diff --git a/src/arm64/sve/compression.cpp b/src/arm64/sve/compression.cpp new file mode 100644 index 0000000..e973183 --- /dev/null +++ b/src/arm64/sve/compression.cpp @@ -0,0 +1,21 @@ +#include + +namespace pernix { +extern "C" { +int sve_compress_block(uint8_t, const float_t*, float_t, uint8_t*) { + return -1; +} + +int sve_compress_block_f64(uint8_t, const double_t*, double_t, uint8_t*) { + return -1; +} + +int sve_compress_blocks(uint8_t, const float_t*, float_t, uint8_t*, uint32_t) { + return -1; +} + +int sve_compress_blocks_f64(uint8_t, const double_t*, double_t, uint8_t*, uint32_t) { + return -1; +} +} +} // namespace pernix diff --git a/src/arm64/sve/decompression.cpp b/src/arm64/sve/decompression.cpp new file mode 100644 index 0000000..c6d84be --- /dev/null +++ b/src/arm64/sve/decompression.cpp @@ -0,0 +1,21 @@ +#include + +namespace pernix { +extern "C" { +int sve_decompress_block(uint8_t, const uint8_t*, float_t, float_t*) { + return -1; +} + +int sve_decompress_block_f64(uint8_t, const uint8_t*, double_t, double_t*) { + return -1; +} + +int sve_decompress_blocks(uint8_t, const uint8_t*, float_t, float_t*, uint32_t) { + return -1; +} + +int sve_decompress_blocks_f64(uint8_t, const uint8_t*, double_t, double_t*, uint32_t) { + return -1; +} +} +} // namespace pernix diff --git a/src/arm64/sve2/compression.cpp b/src/arm64/sve2/compression.cpp new file mode 100644 index 0000000..0a55f16 --- /dev/null +++ b/src/arm64/sve2/compression.cpp @@ -0,0 +1,21 @@ +#include + +namespace pernix { +extern "C" { +int sve2_compress_block(uint8_t, const float_t*, float_t, uint8_t*) { + return -1; +} + +int sve2_compress_block_f64(uint8_t, const double_t*, double_t, uint8_t*) { + return -1; +} + +int sve2_compress_blocks(uint8_t, const float_t*, float_t, uint8_t*, uint32_t) { + return -1; +} + +int sve2_compress_blocks_f64(uint8_t, const double_t*, double_t, uint8_t*, uint32_t) { + return -1; +} +} +} // namespace pernix diff --git a/src/arm64/sve2/decompression.cpp b/src/arm64/sve2/decompression.cpp new file mode 100644 index 0000000..8d170f6 --- /dev/null +++ b/src/arm64/sve2/decompression.cpp @@ -0,0 +1,21 @@ +#include + +namespace pernix { +extern "C" { +int sve2_decompress_block(uint8_t, const uint8_t*, float_t, float_t*) { + return -1; +} + +int sve2_decompress_block_f64(uint8_t, const uint8_t*, double_t, double_t*) { + return -1; +} + +int sve2_decompress_blocks(uint8_t, const uint8_t*, float_t, float_t*, uint32_t) { + return -1; +} + +int sve2_decompress_blocks_f64(uint8_t, const uint8_t*, double_t, double_t*, uint32_t) { + return -1; +} +} +} // namespace pernix diff --git a/src/pernix.cpp b/src/pernix.cpp index 87ccf9d..94d9d14 100644 --- a/src/pernix.cpp +++ b/src/pernix.cpp @@ -6,7 +6,7 @@ extern "C" { #endif // Use the best available implementation based on detected CPU features at compile time -#ifdef PERNIX_AVX2_ENABLED +#if defined(PERNIX_BACKEND_X86) && defined(PERNIX_AVX2_ENABLED) #ifdef PERNIX_AVX512_VBMI_ENABLED int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { return mm512_compress_block_avx512vbmi(bit_width, input, scale, output); @@ -80,6 +80,114 @@ int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ i return mm256_decompress_blocks_f64_avx2(bit_width, input, scale, output, blocks); } #endif +#elif defined(PERNIX_BACKEND_ARM64_NEON) +int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { + return neon_compress_block(bit_width, input, scale, output); +} + +int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { + return neon_compress_block_f64(bit_width, input, scale, output); +} + +int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, + const uint32_t blocks) { + return neon_compress_blocks(bit_width, input, scale, output, blocks); +} + +int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, + const uint32_t blocks) { + return neon_compress_blocks_f64(bit_width, input, scale, output, blocks); +} + +int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + return neon_decompress_block(bit_width, input, scale, output); +} + +int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + return neon_decompress_block_f64(bit_width, input, scale, output); +} + +int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, + const uint32_t blocks) { + return neon_decompress_blocks(bit_width, input, scale, output, blocks); +} + +int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, + const uint32_t blocks) { + return neon_decompress_blocks_f64(bit_width, input, scale, output, blocks); +} +#elif defined(PERNIX_BACKEND_ARM64_SVE) +int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { + return sve_compress_block(bit_width, input, scale, output); +} + +int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { + return sve_compress_block_f64(bit_width, input, scale, output); +} + +int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, + const uint32_t blocks) { + return sve_compress_blocks(bit_width, input, scale, output, blocks); +} + +int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, + const uint32_t blocks) { + return sve_compress_blocks_f64(bit_width, input, scale, output, blocks); +} + +int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + return sve_decompress_block(bit_width, input, scale, output); +} + +int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + return sve_decompress_block_f64(bit_width, input, scale, output); +} + +int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, + const uint32_t blocks) { + return sve_decompress_blocks(bit_width, input, scale, output, blocks); +} + +int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, + const uint32_t blocks) { + return sve_decompress_blocks_f64(bit_width, input, scale, output, blocks); +} +#elif defined(PERNIX_BACKEND_ARM64_SVE2) +int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { + return sve2_compress_block(bit_width, input, scale, output); +} + +int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { + return sve2_compress_block_f64(bit_width, input, scale, output); +} + +int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, + const uint32_t blocks) { + return sve2_compress_blocks(bit_width, input, scale, output, blocks); +} + +int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, + const uint32_t blocks) { + return sve2_compress_blocks_f64(bit_width, input, scale, output, blocks); +} + +int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + return sve2_decompress_block(bit_width, input, scale, output); +} + +int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + return sve2_decompress_block_f64(bit_width, input, scale, output); +} + +int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, + const uint32_t blocks) { + return sve2_decompress_blocks(bit_width, input, scale, output, blocks); +} + +int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, + const uint32_t blocks) { + return sve2_decompress_blocks_f64(bit_width, input, scale, output, blocks); +} #else int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { return compress_block_fallback(bit_width, input, scale, output); @@ -121,4 +229,4 @@ int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ i #ifdef __cplusplus } } // namespace pernix -#endif // __cplusplus \ No newline at end of file +#endif // __cplusplus diff --git a/tests/arm64/neon/.gitkeep b/tests/arm64/neon/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/arm64/sve/.gitkeep b/tests/arm64/sve/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/arm64/sve2/.gitkeep b/tests/arm64/sve2/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/compression/fallback_compression_tests.cpp b/tests/fallback/compression_tests.cpp similarity index 97% rename from tests/compression/fallback_compression_tests.cpp rename to tests/fallback/compression_tests.cpp index 9b50109..78249d7 100644 --- a/tests/compression/fallback_compression_tests.cpp +++ b/tests/fallback/compression_tests.cpp @@ -1,4 +1,4 @@ -#include <../../include/pernix/pernix.h> +#include #include TYPED_TEST(CompressionTest, FallbackCompressBlock) { diff --git a/tests/decompression/fallback_decompression_tests.cpp b/tests/fallback/decompression_tests.cpp similarity index 97% rename from tests/decompression/fallback_decompression_tests.cpp rename to tests/fallback/decompression_tests.cpp index 3c5dc1f..08c26c4 100644 --- a/tests/decompression/fallback_decompression_tests.cpp +++ b/tests/fallback/decompression_tests.cpp @@ -1,4 +1,4 @@ -#include <../../include/pernix/pernix.h> +#include #include TYPED_TEST(DecompressionTest, FallbackDecompressBlock) { diff --git a/tests/fallback_edge_tests.cpp b/tests/fallback/edge_tests.cpp similarity index 100% rename from tests/fallback_edge_tests.cpp rename to tests/fallback/edge_tests.cpp diff --git a/tests/include/testset.h b/tests/include/testset.h index 6535957..5ef4fa1 100644 --- a/tests/include/testset.h +++ b/tests/include/testset.h @@ -1,7 +1,7 @@ #ifndef PERNIX_TESTSET_H #define PERNIX_TESTSET_H -#include <../../include/pernix/pernix.h> +#include #include #include diff --git a/tests/compression/avx2_compression_tests.cpp b/tests/x86/avx2/compression_tests.cpp similarity index 97% rename from tests/compression/avx2_compression_tests.cpp rename to tests/x86/avx2/compression_tests.cpp index 1c2892b..bd7f683 100644 --- a/tests/compression/avx2_compression_tests.cpp +++ b/tests/x86/avx2/compression_tests.cpp @@ -1,4 +1,4 @@ -#include <../../include/pernix/pernix.h> +#include #include #ifdef PERNIX_AVX2_ENABLED diff --git a/tests/decompression/avx2_decompression_tests.cpp b/tests/x86/avx2/decompression_tests.cpp similarity index 97% rename from tests/decompression/avx2_decompression_tests.cpp rename to tests/x86/avx2/decompression_tests.cpp index e0f039f..a6fc2c5 100644 --- a/tests/decompression/avx2_decompression_tests.cpp +++ b/tests/x86/avx2/decompression_tests.cpp @@ -1,4 +1,4 @@ -#include <../../include/pernix/pernix.h> +#include #include #ifdef PERNIX_AVX2_ENABLED diff --git a/tests/compression/avx512vbmi_compression_tests.cpp b/tests/x86/avx512vbmi/compression_tests.cpp similarity index 100% rename from tests/compression/avx512vbmi_compression_tests.cpp rename to tests/x86/avx512vbmi/compression_tests.cpp diff --git a/tests/decompression/avx512vbmi_decompression_tests.cpp b/tests/x86/avx512vbmi/decompression_tests.cpp similarity index 97% rename from tests/decompression/avx512vbmi_decompression_tests.cpp rename to tests/x86/avx512vbmi/decompression_tests.cpp index 446443a..f44dd8d 100644 --- a/tests/decompression/avx512vbmi_decompression_tests.cpp +++ b/tests/x86/avx512vbmi/decompression_tests.cpp @@ -1,4 +1,4 @@ -#include <../../include/pernix/pernix.h> +#include #include #ifdef PERNIX_AVX512_VBMI_ENABLED diff --git a/tests/compression/bmi2_compression_tests.cpp b/tests/x86/bmi2/compression_tests.cpp similarity index 97% rename from tests/compression/bmi2_compression_tests.cpp rename to tests/x86/bmi2/compression_tests.cpp index 85d3cac..b7fc2fd 100644 --- a/tests/compression/bmi2_compression_tests.cpp +++ b/tests/x86/bmi2/compression_tests.cpp @@ -1,4 +1,4 @@ -#include <../../include/pernix/pernix.h> +#include #include #ifdef PERNIX_BMI2_ENABLED diff --git a/tests/decompression/bmi2_decompression_tests.cpp b/tests/x86/bmi2/decompression_tests.cpp similarity index 97% rename from tests/decompression/bmi2_decompression_tests.cpp rename to tests/x86/bmi2/decompression_tests.cpp index 11a8efb..dd7efc1 100644 --- a/tests/decompression/bmi2_decompression_tests.cpp +++ b/tests/x86/bmi2/decompression_tests.cpp @@ -1,4 +1,4 @@ -#include <../../include/pernix/pernix.h> +#include #include #ifdef PERNIX_BMI2_ENABLED From 22f156223ebf5cbaee7d974e3518cc576bdaf958 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Tue, 19 May 2026 17:09:03 +0200 Subject: [PATCH 02/43] Refactor AVX512 compression and decompression headers: improve include organization, address minor formatting inconsistencies, and move utility functions to a shared `utils` file for better maintainability. --- include/pernix/simd_compat.h | 23 +-------- include/pernix/x86/avx512vbmi/compression.h | 51 ++++++++++--------- include/pernix/x86/avx512vbmi/decompression.h | 49 +++++++++--------- include/pernix/x86/utils.h | 16 ++++++ 4 files changed, 70 insertions(+), 69 deletions(-) create mode 100644 include/pernix/x86/utils.h diff --git a/include/pernix/simd_compat.h b/include/pernix/simd_compat.h index 8a66ee7..e96ec30 100644 --- a/include/pernix/simd_compat.h +++ b/include/pernix/simd_compat.h @@ -10,8 +10,8 @@ // #define SIMDE_NO_NATIVE #if defined(PERNIX_BACKEND_X86) #include -#include #include +#include #elif defined(PERNIX_BACKEND_ARM64_NEON) #include #elif defined(PERNIX_BACKEND_ARM64_SVE) || defined(PERNIX_BACKEND_ARM64_SVE2) @@ -47,25 +47,4 @@ #endif #endif -template - requires(std::is_integral_v && sizeof(T) <= 8) -static constexpr T tail_mask(const uint8_t bit_width, const uint32_t remaining_elements) { - const uint32_t tail_bits = remaining_elements * bit_width; - const uint32_t tail_bytes = (tail_bits + 7u) / 8u; - if (tail_bytes == 0u) { - return static_cast(0); - } - if (tail_bytes >= 64u) { - return static_cast(~uint64_t{0}); - } - const uint64_t mask = (uint64_t{1} << tail_bytes) - 1u; - return static_cast(mask); -} - -static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t remaining_elements) { - const uint32_t tail_bits = remaining_elements * bit_width; - const uint32_t tail_bytes = (tail_bits + 7u) / 8u; - return tail_bytes; -} - #endif // PERNIX_SIMD_COMPAT_H diff --git a/include/pernix/x86/avx512vbmi/compression.h b/include/pernix/x86/avx512vbmi/compression.h index bc5a375..e1cb4f0 100644 --- a/include/pernix/x86/avx512vbmi/compression.h +++ b/include/pernix/x86/avx512vbmi/compression.h @@ -1,13 +1,16 @@ #ifndef PERNIX_AVX512VBMI_COMPRESSION_H #define PERNIX_AVX512VBMI_COMPRESSION_H +#include #include -#include #include -#include +#include +#include #include +using namespace pernix::x86::internal; + namespace pernix { namespace internal { template @@ -132,7 +135,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - input += 64; + input += 64; output += 8 * BIT_WIDTH; } } @@ -150,7 +153,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri const __m256i packed = m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_2x128(converted1, converted2)); mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } @@ -162,7 +165,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(converted); mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -207,7 +210,7 @@ template const __m512i packed = m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_2x256(converted1, converted2)); mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } } @@ -220,7 +223,7 @@ template const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(converted); mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -232,7 +235,7 @@ template const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -269,7 +272,7 @@ template const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(packed_input); mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } } @@ -281,14 +284,14 @@ template const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); const __m256i packed_input = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); } @@ -344,7 +347,7 @@ template mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - input += 64; + input += 64; output += 8 * BIT_WIDTH; } } @@ -370,7 +373,7 @@ template mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } @@ -387,7 +390,7 @@ template mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -447,7 +450,7 @@ template mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } } @@ -465,7 +468,7 @@ template mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -477,7 +480,7 @@ template const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -517,7 +520,7 @@ template const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(make_m512i_from_2x256(quantized1, quantized2)); mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } } @@ -529,7 +532,7 @@ template const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -543,7 +546,7 @@ template return 0; } -} // namespace internal +} // namespace internal /** * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. @@ -618,7 +621,7 @@ int mm512_compress_blocks_avx512vbmi(const float_t* __restrict__ input, const fl for (uint32_t block = 0; block < blocks; ++block) { mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } @@ -644,13 +647,13 @@ int mm512_compress_blocks_avx512vbmi(const double_t* __restrict__ input, const d for (uint32_t block = 0; block < blocks; ++block) { mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } return 0; } -} // namespace pernix +} // namespace pernix #ifdef __cplusplus namespace pernix { @@ -716,7 +719,7 @@ int mm512_compress_blocks_f64_avx512vbmi(uint8_t bit_width, const double_t* __re #ifdef __cplusplus } -} // namespace pernix +} // namespace pernix #endif #endif // PERNIX_AVX512VBMI_COMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/decompression.h b/include/pernix/x86/avx512vbmi/decompression.h index 61280c9..6320240 100644 --- a/include/pernix/x86/avx512vbmi/decompression.h +++ b/include/pernix/x86/avx512vbmi/decompression.h @@ -1,13 +1,16 @@ #ifndef PERNIX_AVX512VBMI_DECOMPRESSION_H #define PERNIX_AVX512VBMI_DECOMPRESSION_H +#include #include -#include #include -#include +#include +#include #include +using namespace pernix::x86::internal; + namespace pernix { namespace internal { /** @@ -58,7 +61,7 @@ template _mm512_storeu_ps(output + 48, dequantized4); output += 64; - input += 8 * BIT_WIDTH; + input += 8 * BIT_WIDTH; } } @@ -76,7 +79,7 @@ template _mm512_storeu_ps(output + 16, dequantized2); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } if constexpr (iterations_16 > 0) { @@ -90,7 +93,7 @@ template _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -159,7 +162,7 @@ template _mm512_storeu_pd(output + 56, dequantized8); output += 64; - input += 8 * BIT_WIDTH; + input += 8 * BIT_WIDTH; } if constexpr (iterations_32 > 0) { @@ -185,7 +188,7 @@ template _mm512_storeu_pd(output + 24, dequantized4); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } if constexpr (iterations_16 > 0) { @@ -202,7 +205,7 @@ template _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -255,7 +258,7 @@ template _mm512_storeu_ps(output + 16, dequantized2); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } } @@ -269,7 +272,7 @@ template _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (iterations_8 > 0) { @@ -282,7 +285,7 @@ template _mm256_storeu_ps(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -333,7 +336,7 @@ template _mm512_storeu_pd(output + 24, dequantized4); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } } @@ -351,7 +354,7 @@ template _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (iterations_8 > 0) { @@ -365,7 +368,7 @@ template _mm512_storeu_pd(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -406,7 +409,7 @@ template _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } } @@ -419,7 +422,7 @@ template _mm256_storeu_ps(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -462,7 +465,7 @@ template _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } } @@ -477,7 +480,7 @@ template _mm512_storeu_pd(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -493,7 +496,7 @@ template return 0; } -} // namespace internal +} // namespace internal /** * @brief Decompress a single 512\-bit block using AVX-512 and AVX-512-VBMI instructions. @@ -569,7 +572,7 @@ int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const for (uint32_t block = 0; block < blocks; ++block) { mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -596,12 +599,12 @@ int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const for (uint32_t block = 0; block < blocks; ++block) { mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix +} // namespace pernix #ifdef __cplusplus namespace pernix { @@ -666,7 +669,7 @@ int mm512_decompress_blocks_f64_avx512vbmi(uint8_t bit_width, const uint8_t* __r #ifdef __cplusplus } -} // namespace pernix +} // namespace pernix #endif #endif // PERNIX_AVX512VBMI_DECOMPRESSION_H diff --git a/include/pernix/x86/utils.h b/include/pernix/x86/utils.h new file mode 100644 index 0000000..185e0a2 --- /dev/null +++ b/include/pernix/x86/utils.h @@ -0,0 +1,16 @@ +#ifndef PERNIX_X86_UTILS_H +#define PERNIX_X86_UTILS_H + +#include + +namespace pernix::x86::internal { + +static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t remaining_elements) { + const uint32_t tail_bits = remaining_elements * bit_width; + const uint32_t tail_bytes = (tail_bits + 7u) / 8u; + return tail_bytes; +} + +} // namespace pernix::x86::internal + +#endif // PERNIX_X86_UTILS_H From 37b3a725576985aa6db47107cddb7c539fc67c13 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Tue, 19 May 2026 17:26:44 +0200 Subject: [PATCH 03/43] Replace SIMDe submodule with FetchContent for streamlined dependency management. --- .gitmodules | 3 --- CMakeLists.txt | 16 +++++++++++++++- external/simde | 1 - 3 files changed, 15 insertions(+), 5 deletions(-) delete mode 100644 .gitmodules delete mode 160000 external/simde diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index ae538ec..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "external/simde"] - path = external/simde - url = https://github.com/simd-everywhere/simde diff --git a/CMakeLists.txt b/CMakeLists.txt index 3fa49be..8394abf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,21 @@ endif () list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") if (PERNIX_USE_SIMDE) - add_subdirectory(external/simde EXCLUDE_FROM_ALL) + include(FetchContent) + FetchContent_Declare( + simde + GIT_REPOSITORY https://github.com/simd-everywhere/simde.git + GIT_TAG master + GIT_SHALLOW TRUE + GIT_PROGRESS TRUE + EXCLUDE_FROM_ALL + ) + FetchContent_MakeAvailable(simde) + + if (NOT TARGET simde::simde AND DEFINED simde_SOURCE_DIR AND EXISTS "${simde_SOURCE_DIR}/simde") + add_library(simde::simde INTERFACE IMPORTED GLOBAL) + target_include_directories(simde::simde INTERFACE "${simde_SOURCE_DIR}") + endif () endif () include(CTest) diff --git a/external/simde b/external/simde deleted file mode 160000 index 1a1ca5e..0000000 --- a/external/simde +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1a1ca5ee71518d8a115234dad1e2d871421953b7 From 11779ea1ff10cd012eada302fb5f70041bf17039 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Tue, 19 May 2026 19:07:56 +0200 Subject: [PATCH 04/43] Integrate `CONFIGURE_DEPENDS` in source file globbing, improve CMake configuration with target aliases, install rules, and LTO support, enhance SIMDe handling with flexible provider selection and bundling, and refine compiler flag management for better compatibility. --- CMakeLists.txt | 90 ++++++++++++++++++++++++------------ include/pernix/simd_compat.h | 3 ++ src/CMakeLists.txt | 50 ++++++++++++++++++-- 3 files changed, 109 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8394abf..b8c8208 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,8 @@ option(PERNIX_DISABLE_AVX2 "Disable AVX2 optimizations" off) option(PERNIX_DISABLE_AVX512 "Disable AVX512 optimizations" off) option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" off) +set(PERNIX_SIMDE_PROVIDER "AUTO" CACHE STRING "SIMDe provider when PERNIX_USE_SIMDE is enabled (AUTO, PACKAGE, FETCH)") +set_property(CACHE PERNIX_SIMDE_PROVIDER PROPERTY STRINGS AUTO PACKAGE FETCH) set(PERNIX_ARCH_BACKEND "AUTO" CACHE STRING "Pernix architecture backend (AUTO, FALLBACK, X86, ARM64_NEON, ARM64_SVE, ARM64_SVE2)") set_property(CACHE PERNIX_ARCH_BACKEND PROPERTY STRINGS AUTO FALLBACK X86 ARM64_NEON ARM64_SVE ARM64_SVE2) @@ -24,24 +26,42 @@ if (NOT PERNIX_ARCH_BACKEND IN_LIST PERNIX_VALID_ARCH_BACKENDS) message(FATAL_ERROR "Unsupported PERNIX_ARCH_BACKEND='${PERNIX_ARCH_BACKEND}'. Expected one of: ${PERNIX_VALID_ARCH_BACKENDS}") endif () +string(TOUPPER "${PERNIX_SIMDE_PROVIDER}" PERNIX_SIMDE_PROVIDER) +set(PERNIX_VALID_SIMDE_PROVIDERS AUTO PACKAGE FETCH) +if (NOT PERNIX_SIMDE_PROVIDER IN_LIST PERNIX_VALID_SIMDE_PROVIDERS) + message(FATAL_ERROR "Unsupported PERNIX_SIMDE_PROVIDER='${PERNIX_SIMDE_PROVIDER}'. Expected one of: ${PERNIX_VALID_SIMDE_PROVIDERS}") +endif () + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") +set(PERNIX_BUNDLE_SIMDE_FOR_INSTALL OFF) if (PERNIX_USE_SIMDE) - include(FetchContent) - FetchContent_Declare( - simde - GIT_REPOSITORY https://github.com/simd-everywhere/simde.git - GIT_TAG master - GIT_SHALLOW TRUE - GIT_PROGRESS TRUE - EXCLUDE_FROM_ALL - ) - FetchContent_MakeAvailable(simde) + if (PERNIX_SIMDE_PROVIDER STREQUAL "AUTO" OR PERNIX_SIMDE_PROVIDER STREQUAL "PACKAGE") + find_package(simde CONFIG QUIET) + endif () + + if (NOT TARGET simde::simde AND (PERNIX_SIMDE_PROVIDER STREQUAL "AUTO" OR PERNIX_SIMDE_PROVIDER STREQUAL "FETCH")) + include(FetchContent) + set(SIMDE_TEST_CMAKE_PACKAGING OFF CACHE BOOL "Test SIMDe CMake packaging" FORCE) + FetchContent_Declare( + simde + GIT_REPOSITORY https://github.com/simd-everywhere/simde.git + GIT_TAG f3e8262173b7089db9a9d57a9ecef8dd07ad9c97 + GIT_PROGRESS TRUE + EXCLUDE_FROM_ALL + ) + FetchContent_MakeAvailable(simde) + set(PERNIX_BUNDLE_SIMDE_FOR_INSTALL ON) + endif () if (NOT TARGET simde::simde AND DEFINED simde_SOURCE_DIR AND EXISTS "${simde_SOURCE_DIR}/simde") add_library(simde::simde INTERFACE IMPORTED GLOBAL) target_include_directories(simde::simde INTERFACE "${simde_SOURCE_DIR}") endif () + + if (NOT TARGET simde::simde) + message(FATAL_ERROR "PERNIX_USE_SIMDE is enabled, but simde::simde was not found. Set PERNIX_SIMDE_PROVIDER=FETCH or install SIMDe's CMake package.") + endif () endif () include(CTest) @@ -62,28 +82,42 @@ else () endif () message(STATUS "Pernix version: ${VERSION}, normalized to ${NORMALIZED_VERSION}") -set(BENCHMARK_CXX_STANDARD 20) - -set(CMAKE_CXX_STANDARD ${BENCHMARK_CXX_STANDARD}) -set(CMAKE_CXX_STANDARD_REQUIRED YES) -set(CMAKE_CXX_EXTENSIONS OFF) - -include(AddCXXCompilerFlag) if (MSVC) message(FATAL_ERROR "MSVC compiler is not supported") else () - add_cxx_compiler_flag(-Wall) - add_cxx_compiler_flag(-Wextra) - add_cxx_compiler_flag(-Wshadow) - add_cxx_compiler_flag(-Wfloat-equal) - add_cxx_compiler_flag(-Wold-style-cast) - add_cxx_compiler_flag(-Wconversion) - add_cxx_compiler_flag(-fstrict-aliasing) - add_cxx_compiler_flag(-Wno-ignored-attributes) + include(CheckCXXCompilerFlag) + set(PERNIX_PRIVATE_COMPILE_OPTIONS) + foreach (PERNIX_CXX_FLAG + -Wall + -Wextra + -Wshadow + -Wfloat-equal + -Wold-style-cast + -Wconversion + -fstrict-aliasing + -Wno-ignored-attributes + ) + string(MAKE_C_IDENTIFIER "PERNIX_HAS_CXX_FLAG_${PERNIX_CXX_FLAG}" PERNIX_CXX_FLAG_VARIABLE) + check_cxx_compiler_flag("${PERNIX_CXX_FLAG}" "${PERNIX_CXX_FLAG_VARIABLE}") + if (${PERNIX_CXX_FLAG_VARIABLE}) + list(APPEND PERNIX_PRIVATE_COMPILE_OPTIONS "${PERNIX_CXX_FLAG}") + else () + message(STATUS "Compiler flag not supported: ${PERNIX_CXX_FLAG}") + endif () + endforeach () if (PERNIX_ENABLE_LTO) - add_cxx_compiler_flag(-flto=auto) - add_cxx_compiler_flag(-Wno-lto-type-mismatch) + include(CheckIPOSupported) + check_ipo_supported(RESULT PERNIX_IPO_SUPPORTED OUTPUT PERNIX_IPO_ERROR) + if (NOT PERNIX_IPO_SUPPORTED) + message(FATAL_ERROR "PERNIX_ENABLE_LTO is enabled, but IPO/LTO is not supported: ${PERNIX_IPO_ERROR}") + endif () + + check_cxx_compiler_flag("-Wno-lto-type-mismatch" PERNIX_HAS_CXX_FLAG_WNO_LTO_TYPE_MISMATCH) + if (PERNIX_HAS_CXX_FLAG_WNO_LTO_TYPE_MISMATCH) + list(APPEND PERNIX_PRIVATE_COMPILE_OPTIONS "-Wno-lto-type-mismatch") + endif () + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") find_program(GCC_AR gcc-ar) if (GCC_AR) @@ -106,8 +140,6 @@ else () endif () endif () -include_directories(${PROJECT_SOURCE_DIR}/include) - add_subdirectory(src) if (PERNIX_ENABLE_FORTRAN_BINDINGS) diff --git a/include/pernix/simd_compat.h b/include/pernix/simd_compat.h index e96ec30..f96a84f 100644 --- a/include/pernix/simd_compat.h +++ b/include/pernix/simd_compat.h @@ -7,6 +7,9 @@ #if defined(PERNIX_USE_SIMDE) #define SIMDE_ENABLE_NATIVE_ALIASES #undef SIMDE_X86_AVX512FP16_NATIVE +#if defined(__clang__) +#define SIMDE_X86_AVX512BF16_NATIVE +#endif // #define SIMDE_NO_NATIVE #if defined(PERNIX_BACKEND_X86) #include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f0c7e3e..8c79ab6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,9 @@ include(GNUInstallDirs) +include(CMakePackageConfigHelpers) file(GLOB_RECURSE PERNIX_COMMON_SOURCES + CONFIGURE_DEPENDS ./fallback/*.cpp ./pernix.cpp ${PROJECT_SOURCE_DIR}/include/pernix/*.h @@ -24,6 +26,7 @@ message(STATUS "Pernix architecture backend: ${PERNIX_SELECTED_ARCH_BACKEND}") if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "X86") file(GLOB_RECURSE PERNIX_X86_SOURCES + CONFIGURE_DEPENDS ./x86/*.cpp ${PROJECT_SOURCE_DIR}/include/pernix/x86/*.h ) @@ -31,6 +34,7 @@ if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "X86") elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_NEON") file(GLOB_RECURSE PERNIX_ARM64_NEON_SOURCES + CONFIGURE_DEPENDS ./arm64/neon/*.cpp ${PROJECT_SOURCE_DIR}/include/pernix/arm64/neon/*.h ) @@ -38,6 +42,7 @@ elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_NEON") elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE") file(GLOB_RECURSE PERNIX_ARM64_SVE_SOURCES + CONFIGURE_DEPENDS ./arm64/sve/*.cpp ${PROJECT_SOURCE_DIR}/include/pernix/arm64/sve/*.h ) @@ -45,6 +50,7 @@ elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE") elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE2") file(GLOB_RECURSE PERNIX_ARM64_SVE2_SOURCES + CONFIGURE_DEPENDS ./arm64/sve2/*.cpp ${PROJECT_SOURCE_DIR}/include/pernix/arm64/sve2/*.h ) @@ -52,13 +58,20 @@ elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE2") endif () add_library(pernix SHARED ${PERNIX_SOURCES}) +add_library(pernix::pernix ALIAS pernix) set_target_properties(pernix PROPERTIES OUTPUT_NAME "pernix" VERSION ${NORMALIZED_VERSION} ) +target_compile_features(pernix PUBLIC cxx_std_20) +target_compile_options(pernix PRIVATE ${PERNIX_PRIVATE_COMPILE_OPTIONS}) target_include_directories(pernix PUBLIC $ + $ ) +if (PERNIX_ENABLE_LTO) + set_target_properties(pernix PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) +endif () if (PERNIX_USE_SIMDE) target_link_libraries(pernix PUBLIC simde::simde) @@ -85,23 +98,50 @@ configure_file( if (PERNIX_ENABLE_INSTALL) install(TARGETS pernix + EXPORT pernixTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) - install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/pernix ${PROJECT_BINARY_DIR}/include/pernix + install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/pernix" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.*h" ) + if (PERNIX_USE_SIMDE AND PERNIX_BUNDLE_SIMDE_FOR_INSTALL) + install(DIRECTORY "${simde_SOURCE_DIR}/simde" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + FILES_MATCHING PATTERN "*.h" + ) + endif () + + configure_package_config_file( + "${PROJECT_SOURCE_DIR}/cmake/pernixConfig.cmake.in" + "${PROJECT_BINARY_DIR}/pernixConfig.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/pernix" + ) + write_basic_package_version_file( + "${PROJECT_BINARY_DIR}/pernixConfigVersion.cmake" + VERSION ${NORMALIZED_VERSION} + COMPATIBILITY SameMajorVersion + ) + install( + FILES + "${PROJECT_BINARY_DIR}/pernixConfig.cmake" + "${PROJECT_BINARY_DIR}/pernixConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/pernix" + ) + install( + EXPORT pernixTargets + FILE pernixTargets.cmake + NAMESPACE pernix:: + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/pernix" + ) install( FILES ${PROJECT_BINARY_DIR}/pernix.pc DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" ) - - add_custom_target(uninstall COMMAND xargs rm -vf < ${PROJECT_BINARY_DIR}/install_manifest.txt) endif () if (PERNIX_ENABLE_DOXYGEN) @@ -127,7 +167,7 @@ if (PERNIX_ENABLE_DOXYGEN) WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMENT "Generating documentation with Doxygen." ) - if (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS) + if (PERNIX_ENABLE_INSTALL AND PERNIX_INSTALL_DOCS) install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html/" DESTINATION ${CMAKE_INSTALL_DOCDIR}) endif () From 17807c8ca2a62158d86e2154214e05ed0acb5736 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Tue, 19 May 2026 20:11:11 +0200 Subject: [PATCH 05/43] Refactor inline attributes in AVX512 compression and decompression headers for consistency and clarity --- include/pernix/x86/avx512vbmi/compression.h | 62 ++++++++-------- include/pernix/x86/avx512vbmi/decompression.h | 72 +++++++++---------- include/pernix/x86/avx512vbmi/packing.h | 18 ++--- include/pernix/x86/avx512vbmi/tables.h | 66 +++++++++-------- 4 files changed, 108 insertions(+), 110 deletions(-) diff --git a/include/pernix/x86/avx512vbmi/compression.h b/include/pernix/x86/avx512vbmi/compression.h index e1cb4f0..9621e12 100644 --- a/include/pernix/x86/avx512vbmi/compression.h +++ b/include/pernix/x86/avx512vbmi/compression.h @@ -135,7 +135,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - input += 64; + input += 64; output += 8 * BIT_WIDTH; } } @@ -153,7 +153,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri const __m256i packed = m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_2x128(converted1, converted2)); mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } @@ -165,7 +165,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(converted); mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -183,8 +183,8 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_9to16(const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_9to16(const float_t* __restrict__ input, const float_t scale, + uint8_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_32 = elements_per_block / 32; @@ -210,7 +210,7 @@ template const __m512i packed = m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_2x256(converted1, converted2)); mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } } @@ -223,7 +223,7 @@ template const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(converted); mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -235,7 +235,7 @@ template const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -253,8 +253,8 @@ template template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_17to24(const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_17to24(const float_t* __restrict__ input, const float_t scale, + uint8_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_16 = elements_per_block / 16; @@ -272,7 +272,7 @@ template const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(packed_input); mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } } @@ -284,7 +284,7 @@ template const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -301,8 +301,8 @@ template template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_1to8(const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_1to8(const double_t* __restrict__ input, const double_t scale, + uint8_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_64 = elements_per_block / 64; @@ -347,7 +347,7 @@ template mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - input += 64; + input += 64; output += 8 * BIT_WIDTH; } } @@ -373,7 +373,7 @@ template mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } @@ -390,7 +390,7 @@ template mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -416,8 +416,8 @@ template template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_9to16(const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_9to16(const double_t* __restrict__ input, const double_t scale, + uint8_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_32 = elements_per_block / 32; @@ -450,7 +450,7 @@ template mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } } @@ -468,7 +468,7 @@ template mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -480,7 +480,7 @@ template const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -498,8 +498,8 @@ template template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_17to24(const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_17to24(const double_t* __restrict__ input, const double_t scale, + uint8_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_16 = elements_per_block / 16; @@ -520,7 +520,7 @@ template const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(make_m512i_from_2x256(quantized1, quantized2)); mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } } @@ -532,7 +532,7 @@ template const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -546,7 +546,7 @@ template return 0; } -} // namespace internal +} // namespace internal /** * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. @@ -621,7 +621,7 @@ int mm512_compress_blocks_avx512vbmi(const float_t* __restrict__ input, const fl for (uint32_t block = 0; block < blocks; ++block) { mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } @@ -647,13 +647,13 @@ int mm512_compress_blocks_avx512vbmi(const double_t* __restrict__ input, const d for (uint32_t block = 0; block < blocks; ++block) { mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } return 0; } -} // namespace pernix +} // namespace pernix #ifdef __cplusplus namespace pernix { @@ -719,7 +719,7 @@ int mm512_compress_blocks_f64_avx512vbmi(uint8_t bit_width, const double_t* __re #ifdef __cplusplus } -} // namespace pernix +} // namespace pernix #endif #endif // PERNIX_AVX512VBMI_COMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/decompression.h b/include/pernix/x86/avx512vbmi/decompression.h index 6320240..08abc35 100644 --- a/include/pernix/x86/avx512vbmi/decompression.h +++ b/include/pernix/x86/avx512vbmi/decompression.h @@ -16,20 +16,20 @@ namespace internal { /** * @brief Dequantize sixteen integer values to floats. */ -[[gnu::always_inline]] inline __m512 mm512_dequantize_epi32(const __m512i& input, const __m512& scale) { +__always_inline __m512 mm512_dequantize_epi32(const __m512i& input, const __m512& scale) { const __m512 converted = _mm512_cvtepi32_ps(input); return _mm512_mul_ps(converted, scale); } -[[gnu::always_inline]] inline __m512d mm512_dequantize_epi64(const __m512i& input, const __m512d& scale) { +__always_inline __m512d mm512_dequantize_epi64(const __m512i& input, const __m512d& scale) { const __m512d converted = _mm512_cvtepi64_pd(input); return _mm512_mul_pd(converted, scale); } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const uint32_t iterations_64 = elements_per_block / 64; @@ -61,7 +61,7 @@ template _mm512_storeu_ps(output + 48, dequantized4); output += 64; - input += 8 * BIT_WIDTH; + input += 8 * BIT_WIDTH; } } @@ -79,7 +79,7 @@ template _mm512_storeu_ps(output + 16, dequantized2); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } if constexpr (iterations_16 > 0) { @@ -93,7 +93,7 @@ template _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -112,8 +112,8 @@ template template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const uint32_t iterations_64 = elements_per_block / 64; @@ -162,7 +162,7 @@ template _mm512_storeu_pd(output + 56, dequantized8); output += 64; - input += 8 * BIT_WIDTH; + input += 8 * BIT_WIDTH; } if constexpr (iterations_32 > 0) { @@ -188,7 +188,7 @@ template _mm512_storeu_pd(output + 24, dequantized4); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } if constexpr (iterations_16 > 0) { @@ -205,7 +205,7 @@ template _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -230,8 +230,8 @@ template template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_32 = elements_per_block / 32; @@ -258,7 +258,7 @@ template _mm512_storeu_ps(output + 16, dequantized2); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } } @@ -272,7 +272,7 @@ template _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (iterations_8 > 0) { @@ -285,7 +285,7 @@ template _mm256_storeu_ps(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -303,8 +303,8 @@ template template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_32 = elements_per_block / 32; @@ -336,7 +336,7 @@ template _mm512_storeu_pd(output + 24, dequantized4); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } } @@ -354,7 +354,7 @@ template _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (iterations_8 > 0) { @@ -368,7 +368,7 @@ template _mm512_storeu_pd(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -387,8 +387,8 @@ template template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_16 = elements_per_block / 16; @@ -409,7 +409,7 @@ template _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } } @@ -422,7 +422,7 @@ template _mm256_storeu_ps(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -439,8 +439,8 @@ template template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_16 = elements_per_block / 16; @@ -465,7 +465,7 @@ template _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } } @@ -480,7 +480,7 @@ template _mm512_storeu_pd(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -496,7 +496,7 @@ template return 0; } -} // namespace internal +} // namespace internal /** * @brief Decompress a single 512\-bit block using AVX-512 and AVX-512-VBMI instructions. @@ -572,7 +572,7 @@ int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const for (uint32_t block = 0; block < blocks; ++block) { mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -599,15 +599,15 @@ int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const for (uint32_t block = 0; block < blocks; ++block) { mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix +} // namespace pernix -#ifdef __cplusplus namespace pernix { +#ifdef __cplusplus extern "C" { #endif /** @@ -669,7 +669,7 @@ int mm512_decompress_blocks_f64_avx512vbmi(uint8_t bit_width, const uint8_t* __r #ifdef __cplusplus } -} // namespace pernix #endif +} // namespace pernix #endif // PERNIX_AVX512VBMI_DECOMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/packing.h b/include/pernix/x86/avx512vbmi/packing.h index c9f9db9..ba3b132 100644 --- a/include/pernix/x86/avx512vbmi/packing.h +++ b/include/pernix/x86/avx512vbmi/packing.h @@ -11,7 +11,7 @@ namespace m128 { */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -[[gnu::always_inline]] inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { +__always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { if constexpr (BIT_WIDTH == 16) { return input; } else { @@ -48,7 +48,7 @@ template */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -[[gnu::always_inline]] inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { +__always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { if constexpr (BIT_WIDTH == 8) { return input; } else { @@ -93,7 +93,7 @@ template */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -[[gnu::always_inline]] inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i& input) { +__always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i& input) { using tables = pack_tables_avx512_24; const __m128i maskv = _mm_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); @@ -117,7 +117,7 @@ namespace m256 { */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -[[gnu::always_inline]] inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) { +__always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) { if constexpr (BIT_WIDTH == 16) { return input; } else { @@ -154,7 +154,7 @@ template */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -[[gnu::always_inline]] inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { +__always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { if constexpr (BIT_WIDTH == 8) { return input; } else { @@ -199,7 +199,7 @@ template */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -[[gnu::always_inline]] inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i& input) { +__always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i& input) { using tables = pack_tables_avx512_24; const __m256i maskv = _mm256_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); @@ -223,7 +223,7 @@ namespace m512 { */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -[[gnu::always_inline]] inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) { +__always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) { if constexpr (BIT_WIDTH == 16) { return input; } else { @@ -260,7 +260,7 @@ template */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -[[gnu::always_inline]] inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { +__always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { if constexpr (BIT_WIDTH == 8) { return input; } else { @@ -305,7 +305,7 @@ template */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -[[gnu::always_inline]] inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i& input) { +__always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i& input) { using tables = pack_tables_avx512_24; const __m512i maskv = _mm512_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); diff --git a/include/pernix/x86/avx512vbmi/tables.h b/include/pernix/x86/avx512vbmi/tables.h index 9115625..4d66727 100644 --- a/include/pernix/x86/avx512vbmi/tables.h +++ b/include/pernix/x86/avx512vbmi/tables.h @@ -9,9 +9,8 @@ #include namespace pernix::internal { - template -[[gnu::always_inline]] static inline Vec load_table(const std::array& table) { +static __always_inline Vec load_table(const std::array& table) { static_assert(sizeof(table) >= sizeof(Vec), "table is smaller than requested SIMD vector"); if constexpr (std::is_same_v) { return _mm512_load_si512(static_cast(table.data())); @@ -529,13 +528,13 @@ struct pack_tables_avx512_16 { // clang-format on } - [[gnu::always_inline]] static inline Vec get_permute1() { return load_table(permute1); } - [[gnu::always_inline]] static inline Vec get_permute2() { return load_table(permute2); } - [[gnu::always_inline]] static inline Vec get_permute3() { return load_table(permute3); } + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } + static __always_inline Vec get_permute3() { return load_table(permute3); } - [[gnu::always_inline]] static inline Vec get_shift1() { return load_table(shift1); } - [[gnu::always_inline]] static inline Vec get_shift2() { return load_table(shift2); } - [[gnu::always_inline]] static inline Vec get_shift3() { return load_table(shift3); } + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } + static __always_inline Vec get_shift3() { return load_table(shift3); } }; template @@ -591,7 +590,7 @@ struct pack_tables_avx512_24 { return plan; } - static inline constexpr std::array word_plans = [] { + static constexpr std::array word_plans = [] { std::array plans{}; for (uint32_t i = 0; i < 16; ++i) { plans[i] = create_plan(i); @@ -600,7 +599,7 @@ struct pack_tables_avx512_24 { }(); template - [[gnu::always_inline]] static constexpr std::array make_table(Getter getter) { + static __always_inline constexpr std::array make_table(Getter getter) { std::array values{}; for (uint32_t i = 0; i < 16; ++i) { values[i] = getter(word_plans[i]); @@ -608,26 +607,26 @@ struct pack_tables_avx512_24 { return values; } - alignas(64) static inline constexpr auto permute1 = make_table([](const word_plan& p) { return p.left_index1; }); + alignas(64) static constexpr auto permute1 = make_table([](const word_plan& p) { return p.left_index1; }); - alignas(64) static inline constexpr auto permute2 = make_table([](const word_plan& p) { return p.left_index2; }); + alignas(64) static constexpr auto permute2 = make_table([](const word_plan& p) { return p.left_index2; }); - alignas(64) static inline constexpr auto permute3 = make_table([](const word_plan& p) { return p.right_index; }); + alignas(64) static constexpr auto permute3 = make_table([](const word_plan& p) { return p.right_index; }); - alignas(64) static inline constexpr auto shift1 = make_table([](const word_plan& p) { return p.left_shift1; }); + alignas(64) static constexpr auto shift1 = make_table([](const word_plan& p) { return p.left_shift1; }); - alignas(64) static inline constexpr auto shift2 = make_table([](const word_plan& p) { return p.left_shift2; }); + alignas(64) static constexpr auto shift2 = make_table([](const word_plan& p) { return p.left_shift2; }); - alignas(64) static inline constexpr auto shift3 = make_table([](const word_plan& p) { return p.right_shift; }); + alignas(64) static constexpr auto shift3 = make_table([](const word_plan& p) { return p.right_shift; }); public: - [[gnu::always_inline]] static inline Vec get_permute1() { return load_table(permute1); } - [[gnu::always_inline]] static inline Vec get_permute2() { return load_table(permute2); } - [[gnu::always_inline]] static inline Vec get_permute3() { return load_table(permute3); } + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } + static __always_inline Vec get_permute3() { return load_table(permute3); } - [[gnu::always_inline]] static inline Vec get_shift1() { return load_table(shift1); } - [[gnu::always_inline]] static inline Vec get_shift2() { return load_table(shift2); } - [[gnu::always_inline]] static inline Vec get_shift3() { return load_table(shift3); } + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } + static __always_inline Vec get_shift3() { return load_table(shift3); } }; template @@ -693,11 +692,11 @@ struct unpack_tables_avx512_8 { }(); public: - [[gnu::always_inline]] static inline Vec get_permute1() { return load_table(permute1); } - [[gnu::always_inline]] static inline Vec get_permute2() { return load_table(permute2); } + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } - [[gnu::always_inline]] static inline Vec get_shift1() { return load_table(shift1); } - [[gnu::always_inline]] static inline Vec get_shift2() { return load_table(shift2); } + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } }; template @@ -768,11 +767,11 @@ struct unpack_tables_avx512_16 { }(); public: - [[gnu::always_inline]] static inline Vec get_permute1() { return load_table(permute1); } - [[gnu::always_inline]] static inline Vec get_permute2() { return load_table(permute2); } + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } - [[gnu::always_inline]] static inline Vec get_shift1() { return load_table(shift1); } - [[gnu::always_inline]] static inline Vec get_shift2() { return load_table(shift2); } + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } }; template @@ -813,10 +812,9 @@ struct unpack_tables_avx512_24 { }(); public: - [[gnu::always_inline]] static inline Vec get_permute() { return load_table(permute); } - [[gnu::always_inline]] static inline Vec get_shift() { return load_table(shift); } + static __always_inline Vec get_permute() { return load_table(permute); } + static __always_inline Vec get_shift() { return load_table(shift); } }; - -} // namespace pernix::internal +} // namespace pernix::internal #endif // PERNIX_AVX512VBMI_TABLES_H From 552cea37204a02b36085becafc9da57be3a74b64 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Tue, 19 May 2026 20:18:14 +0200 Subject: [PATCH 06/43] Refactor ARM64 NEON and SVE compression/decompression headers: organize includes, introduce namespaced utility functions, add templates for bit-width handling, and update function signatures for consistency. --- include/pernix/arm64/neon/compression.h | 124 ++++++++++++++++++---- include/pernix/arm64/neon/decompression.h | 120 +++++++++++++++++---- include/pernix/arm64/neon/packing.h | 2 - include/pernix/arm64/neon/unpacking.h | 10 -- include/pernix/arm64/sve/compression.h | 124 ++++++++++++++++++---- include/pernix/arm64/sve/decompression.h | 120 +++++++++++++++++---- include/pernix/arm64/sve/packing.h | 2 - include/pernix/arm64/sve/unpacking.h | 2 - 8 files changed, 408 insertions(+), 96 deletions(-) diff --git a/include/pernix/arm64/neon/compression.h b/include/pernix/arm64/neon/compression.h index 65ea786..6e49348 100644 --- a/include/pernix/arm64/neon/compression.h +++ b/include/pernix/arm64/neon/compression.h @@ -2,58 +2,140 @@ #define PERNIX_ARM64_NEON_COMPRESSION_H #include +#include #include #include -namespace pernix { +namespace pernix::arm64::neon { namespace internal { -template -inline constexpr bool neon_compression_unimplemented_v = false; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} } // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_compress_block(const float_t*, float_t, uint8_t*) { - static_assert(internal::neon_compression_unimplemented_v, "ARM64 NEON compression is not implemented yet"); - return -1; +__always_inline int neon_compress_block(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_compress_block_17to24(input, scale, output); + } + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_compress_block(const double_t*, double_t, uint8_t*) { - static_assert(internal::neon_compression_unimplemented_v, "ARM64 NEON compression is not implemented yet"); - return -1; +__always_inline int neon_compress_block(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_compress_block_17to24(input, scale, output); + } + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_compress_blocks(const float_t*, float_t, uint8_t*, uint32_t) { - static_assert(internal::neon_compression_unimplemented_v, "ARM64 NEON compression is not implemented yet"); - return -1; +int neon_compress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, + const uint32_t blocks) { + const uint8_t* block_input = input; + float_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + neon_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_compress_blocks(const double_t*, double_t, uint8_t*, uint32_t) { - static_assert(internal::neon_compression_unimplemented_v, "ARM64 NEON compression is not implemented yet"); - return -1; +int neon_compress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, + const uint32_t blocks) { + const uint8_t* block_input = input; + double_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + neon_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + return 0; } #ifdef __cplusplus extern "C" { #endif -int neon_compress_block(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); -int neon_compress_block_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); -int neon_compress_blocks(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, +int neon_compress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); + + +int neon_compress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, + double_t* __restrict__ output); + +int neon_compress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); -int neon_compress_blocks_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); + +int neon_compress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, + double_t* __restrict__ output, uint32_t blocks); #ifdef __cplusplus } #endif -} // namespace pernix +} // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_COMPRESSION_H diff --git a/include/pernix/arm64/neon/decompression.h b/include/pernix/arm64/neon/decompression.h index 0f8d79e..44744ce 100644 --- a/include/pernix/arm64/neon/decompression.h +++ b/include/pernix/arm64/neon/decompression.h @@ -2,42 +2,119 @@ #define PERNIX_ARM64_NEON_DECOMPRESSION_H #include +#include #include #include -namespace pernix { +namespace pernix::arm64::neon { namespace internal { -template -inline constexpr bool neon_decompression_unimplemented_v = false; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} } // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_block(const uint8_t*, float_t, float_t*) { - static_assert(internal::neon_decompression_unimplemented_v, "ARM64 NEON decompression is not implemented yet"); - return -1; +__always_inline int neon_decompress_block(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_decompress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_decompress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_decompress_block_17to24(input, scale, output); + } + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_block(const uint8_t*, double_t, double_t*) { - static_assert(internal::neon_decompression_unimplemented_v, "ARM64 NEON decompression is not implemented yet"); - return -1; +__always_inline int neon_decompress_block(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_decompress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_decompress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_decompress_block_17to24(input, scale, output); + } + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_blocks(const uint8_t*, float_t, float_t*, uint32_t) { - static_assert(internal::neon_decompression_unimplemented_v, "ARM64 NEON decompression is not implemented yet"); - return -1; +int neon_decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, + const uint32_t blocks) { + const uint8_t* block_input = input; + float_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + neon_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_blocks(const uint8_t*, double_t, double_t*, uint32_t) { - static_assert(internal::neon_decompression_unimplemented_v, "ARM64 NEON decompression is not implemented yet"); - return -1; +int neon_decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, + const uint32_t blocks) { + const uint8_t* block_input = input; + double_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + neon_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + return 0; } #ifdef __cplusplus @@ -45,15 +122,20 @@ extern "C" { #endif int neon_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); -int neon_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); + + +int neon_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, + double_t* __restrict__ output); + int neon_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); -int neon_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); + +int neon_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, + double_t* __restrict__ output, uint32_t blocks); #ifdef __cplusplus } #endif -} // namespace pernix +} // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_DECOMPRESSION_H diff --git a/include/pernix/arm64/neon/packing.h b/include/pernix/arm64/neon/packing.h index c1c8119..538b5a8 100644 --- a/include/pernix/arm64/neon/packing.h +++ b/include/pernix/arm64/neon/packing.h @@ -4,8 +4,6 @@ #include namespace pernix::arm64::neon::internal { -template -inline constexpr bool packing_unimplemented_v = false; } // namespace pernix::arm64::neon::internal #endif // PERNIX_ARM64_NEON_PACKING_H diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h index 32dc5de..ea22b24 100644 --- a/include/pernix/arm64/neon/unpacking.h +++ b/include/pernix/arm64/neon/unpacking.h @@ -4,17 +4,7 @@ #include namespace pernix::arm64::neon::internal { - -template -inline constexpr bool unpacking_unimplemented_v = false; - -namespace b64 { - -} // namespace b64 - - } // namespace pernix::arm64::neon::internal - #endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/arm64/sve/compression.h b/include/pernix/arm64/sve/compression.h index f8abfed..cf83ce0 100644 --- a/include/pernix/arm64/sve/compression.h +++ b/include/pernix/arm64/sve/compression.h @@ -2,58 +2,140 @@ #define PERNIX_ARM64_SVE_COMPRESSION_H #include +#include #include #include -namespace pernix { +namespace pernix::arm64::sve { namespace internal { -template -inline constexpr bool sve_compression_unimplemented_v = false; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_compress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_compress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_compress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_compress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_compress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_compress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} } // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_compress_block(const float_t*, float_t, uint8_t*) { - static_assert(internal::sve_compression_unimplemented_v, "ARM64 SVE compression is not implemented yet"); - return -1; +__always_inline int sve_compress_block(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::sve_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::sve_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::sve_compress_block_17to24(input, scale, output); + } + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_compress_block(const double_t*, double_t, uint8_t*) { - static_assert(internal::sve_compression_unimplemented_v, "ARM64 SVE compression is not implemented yet"); - return -1; +__always_inline int sve_compress_block(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::sve_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::sve_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::sve_compress_block_17to24(input, scale, output); + } + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_compress_blocks(const float_t*, float_t, uint8_t*, uint32_t) { - static_assert(internal::sve_compression_unimplemented_v, "ARM64 SVE compression is not implemented yet"); - return -1; +int sve_compress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, + const uint32_t blocks) { + const uint8_t* block_input = input; + float_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + sve_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_compress_blocks(const double_t*, double_t, uint8_t*, uint32_t) { - static_assert(internal::sve_compression_unimplemented_v, "ARM64 SVE compression is not implemented yet"); - return -1; +int sve_compress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, + const uint32_t blocks) { + const uint8_t* block_input = input; + double_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + sve_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + return 0; } #ifdef __cplusplus extern "C" { #endif -int sve_compress_block(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); -int sve_compress_block_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); -int sve_compress_blocks(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, +int sve_compress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); + + +int sve_compress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, + double_t* __restrict__ output); + +int sve_compress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); -int sve_compress_blocks_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); + +int sve_compress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, + double_t* __restrict__ output, uint32_t blocks); #ifdef __cplusplus } #endif -} // namespace pernix +} // namespace pernix::arm64::sve #endif // PERNIX_ARM64_SVE_COMPRESSION_H diff --git a/include/pernix/arm64/sve/decompression.h b/include/pernix/arm64/sve/decompression.h index 784c0c8..052a3e4 100644 --- a/include/pernix/arm64/sve/decompression.h +++ b/include/pernix/arm64/sve/decompression.h @@ -2,42 +2,119 @@ #define PERNIX_ARM64_SVE_DECOMPRESSION_H #include +#include #include #include -namespace pernix { +namespace pernix::arm64::sve { namespace internal { -template -inline constexpr bool sve_decompression_unimplemented_v = false; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} } // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_decompress_block(const uint8_t*, float_t, float_t*) { - static_assert(internal::sve_decompression_unimplemented_v, "ARM64 SVE decompression is not implemented yet"); - return -1; +__always_inline int sve_decompress_block(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::sve_decompress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::sve_decompress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::sve_decompress_block_17to24(input, scale, output); + } + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_decompress_block(const uint8_t*, double_t, double_t*) { - static_assert(internal::sve_decompression_unimplemented_v, "ARM64 SVE decompression is not implemented yet"); - return -1; +__always_inline int sve_decompress_block(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::sve_decompress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::sve_decompress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::sve_decompress_block_17to24(input, scale, output); + } + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_decompress_blocks(const uint8_t*, float_t, float_t*, uint32_t) { - static_assert(internal::sve_decompression_unimplemented_v, "ARM64 SVE decompression is not implemented yet"); - return -1; +int sve_decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, + const uint32_t blocks) { + const uint8_t* block_input = input; + float_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + sve_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_decompress_blocks(const uint8_t*, double_t, double_t*, uint32_t) { - static_assert(internal::sve_decompression_unimplemented_v, "ARM64 SVE decompression is not implemented yet"); - return -1; +int sve_decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, + const uint32_t blocks) { + const uint8_t* block_input = input; + double_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + sve_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + return 0; } #ifdef __cplusplus @@ -45,15 +122,20 @@ extern "C" { #endif int sve_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); -int sve_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); + + +int sve_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, + double_t* __restrict__ output); + int sve_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); -int sve_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); + +int sve_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, + double_t* __restrict__ output, uint32_t blocks); #ifdef __cplusplus } #endif -} // namespace pernix +} // namespace pernix::arm64::sve #endif // PERNIX_ARM64_SVE_DECOMPRESSION_H diff --git a/include/pernix/arm64/sve/packing.h b/include/pernix/arm64/sve/packing.h index fce21ca..ab57b4f 100644 --- a/include/pernix/arm64/sve/packing.h +++ b/include/pernix/arm64/sve/packing.h @@ -4,8 +4,6 @@ #include namespace pernix::arm64::sve::internal { -template -inline constexpr bool packing_unimplemented_v = false; } // namespace pernix::arm64::sve::internal #endif // PERNIX_ARM64_SVE_PACKING_H diff --git a/include/pernix/arm64/sve/unpacking.h b/include/pernix/arm64/sve/unpacking.h index 3de49ca..2565ab7 100644 --- a/include/pernix/arm64/sve/unpacking.h +++ b/include/pernix/arm64/sve/unpacking.h @@ -4,8 +4,6 @@ #include namespace pernix::arm64::sve::internal { -template -inline constexpr bool unpacking_unimplemented_v = false; } // namespace pernix::arm64::sve::internal #endif // PERNIX_ARM64_SVE_UNPACKING_H From 23553012600b6866bdda71d4a7c551310d29ea10 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Tue, 19 May 2026 21:49:41 +0200 Subject: [PATCH 07/43] WIP: Implement NEON decompression functions with common utilities and templates --- include/pernix/arm64/neon/common.h | 106 ++++++++++++++++++++++ include/pernix/arm64/neon/decompression.h | 106 +++++++++++++++++++--- include/pernix/arm64/neon/unpacking.h | 20 +++- include/pernix/simd_compat.h | 2 +- 4 files changed, 219 insertions(+), 15 deletions(-) create mode 100644 include/pernix/arm64/neon/common.h diff --git a/include/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h new file mode 100644 index 0000000..6efa777 --- /dev/null +++ b/include/pernix/arm64/neon/common.h @@ -0,0 +1,106 @@ +#ifndef PERNIX_ARM64_NEON_COMMON_H +#define PERNIX_ARM64_NEON_COMMON_H + +#include +#include + +namespace pernix::arm64::neon::internal { +static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t remaining_elements) { + const uint32_t tail_bits = remaining_elements * bit_width; + const uint32_t tail_bytes = (tail_bits + 7u) / 8u; + return tail_bytes; +} + +__always_inline int32x4x4_t neon_convert_int8x16_int32x4x2_t(const int8x16_t& input) { + const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); + const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); + + return {{ + vmovl_s16(vget_low_s16(s16_lo)), + vmovl_s16(vget_high_s16(s16_lo)), + vmovl_s16(vget_low_s16(s16_hi)), + vmovl_s16(vget_high_s16(s16_hi)), + }}; +} + +__always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, const float32x4_t& scale) { + return {{ + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), + }}; +} + +__always_inline uint8x16_t neon_load_tail_elements_int8(const uint8_t* input, const uint32_t tail_elements) { + uint8_t buffer[16] = {0}; + std::memcpy(buffer, input, tail_elements * sizeof(uint8_t)); + return vld1q_u8(buffer); +} + +__always_inline uint16x8_t neon_load_tail_elements_int16(const uint8_t* input, const uint32_t tail_elements) { + uint16_t buffer[8] = {0}; + std::memcpy(buffer, input, tail_elements * sizeof(uint16_t)); + return vld1q_u16(buffer); +} + +__always_inline uint32x4_t neon_load_tail_elements_int32(const uint8_t* input, const uint32_t tail_elements) { + uint32_t buffer[4] = {0}; + std::memcpy(buffer, input, tail_elements * sizeof(uint32_t)); + return vld1q_u32(buffer); +} + +__always_inline float32x4_t neon_load_tail_elements_f32(const uint8_t* input, const uint32_t tail_elements) { + float32_t buffer[4] = {0.0f}; + std::memcpy(buffer, input, tail_elements * sizeof(float32_t)); + return vld1q_f32(buffer); +} + +__always_inline float64x2_t neon_load_tail_elements_f64(const uint8_t* input, const uint32_t tail_elements) { + float64_t buffer[2] = {0.0}; + std::memcpy(buffer, input, tail_elements * sizeof(float64_t)); + return vld1q_f64(buffer); +} + +__always_inline void neon_store_tail_elements_int8(uint8_t* output, const uint8x16x4_t& data, const uint32_t tail_elements) { + uint8_t buffer[16 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_u8(buffer + i * 16, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(uint8_t)); +} + +__always_inline void neon_store_tail_elements_int16(uint16_t* output, const uint16x8x4_t& data, const uint32_t tail_elements) { + uint16_t buffer[8 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_u16(buffer + i * 8, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(uint16_t)); +} + +__always_inline void neon_store_tail_elements_int32(uint32_t* output, const uint32x4x4_t& data, const uint32_t tail_elements) { + uint32_t buffer[4 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_u32(buffer + i * 4, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(uint32_t)); +} + +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x4_t& data, const uint32_t tail_elements) { + float32_t buffer[16 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_f32(buffer + i * 4, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); +} + +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x4_t& data, const uint32_t tail_elements) { + float64_t buffer[8 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); +} +} // namespace pernix::arm64::neon::internal + +#endif //PERNIX_ARM64_NEON_COMMON_H \ No newline at end of file diff --git a/include/pernix/arm64/neon/decompression.h b/include/pernix/arm64/neon/decompression.h index 44744ce..90ed78b 100644 --- a/include/pernix/arm64/neon/decompression.h +++ b/include/pernix/arm64/neon/decompression.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -13,48 +14,129 @@ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_16 = elements_per_block / 16; + constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16; + + const float32x4_t scale_v = vdupq_n_f32(scale); + + for (uint32_t i = 0; i < iterations_16; ++i) { + const uint8x16_t source = vld1q_u8(input); + const int8x16_t unpacked = b128::neon_unpack_epi8_1to8(source); + + const int32x4x4_t converted = neon_convert_int8x16_int32x4x2_t(unpacked); + const float32x4x4_t dequantized = neon_dequantize_epi32(converted, scale_v); + + for (uint32_t j = 0; j < 4; ++j) { + vst1q_f32(output, dequantized.val[j]); + output += 4; + } + + input += 2 * BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const uint8x16_t tail_source = neon_load_tail_elements_int8(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8(tail_source); + + const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x2_t(tail_unpacked); + const float32x4x4_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); + + neon_store_tail_elements_f32(output, tail_dequantized, remaining_elements); + } + + return 0; } template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_8 = elements_per_block / 8; + constexpr uint32_t remaining_elements = elements_per_block - iterations_8 * 8; + + for (uint32_t i = 0; i < iterations_8; ++i) { + static_assert(true, "Not yet implemented"); + } + + if constexpr (remaining_elements > 0) { + static_assert(true, "Not yet implemented"); + } } template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_4 = elements_per_block / 4; + constexpr uint32_t remaining_elements = elements_per_block - iterations_4 * 4; + + for (uint32_t i = 0; i < iterations_4; ++i) { + static_assert(true, "Not yet implemented"); + } + + if constexpr (remaining_elements > 0) { + static_assert(true, "Not yet implemented"); + } } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_8 = elements_per_block / 8; + constexpr uint32_t remaining_elements = elements_per_block - iterations_8 * 8; + + for (uint32_t i = 0; i < iterations_8; ++i) { + static_assert(true, "Not yet implemented"); + } + + if constexpr (remaining_elements > 0) { + static_assert(true, "Not yet implemented"); + } } template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_4 = elements_per_block / 4; + constexpr uint32_t remaining_elements = elements_per_block - iterations_4 * 4; + + for (uint32_t i = 0; i < iterations_4; ++i) { + static_assert(true, "Not yet implemented"); + } + + if constexpr (remaining_elements > 0) { + static_assert(true, "Not yet implemented"); + } } template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_2 = elements_per_block / 2; + constexpr uint32_t remaining_elements = elements_per_block - iterations_2 * 2; + + for (uint32_t i = 0; i < iterations_2; ++i) { + static_assert(true, "Not yet implemented"); + } + + if constexpr (remaining_elements > 0) { + static_assert(true, "Not yet implemented"); + } } } // namespace internal diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h index ea22b24..c1ae78f 100644 --- a/include/pernix/arm64/neon/unpacking.h +++ b/include/pernix/arm64/neon/unpacking.h @@ -3,8 +3,24 @@ #include -namespace pernix::arm64::neon::internal { -} // namespace pernix::arm64::neon::internal +namespace pernix::arm64::neon::internal::b128 { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline int8x16_t neon_unpack_epi8_1to8(const int8x16_t& input) { + return input; +} +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline int16x8_t neon_unpack_epi8_9to16(const int16x8_t& input) { + return input; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline int32x4_t neon_unpack_epi8_17to24(const int32x4_t& input) { + return input; +} +} // namespace pernix::arm64::neon::internal::b128 #endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/simd_compat.h b/include/pernix/simd_compat.h index f96a84f..07d5110 100644 --- a/include/pernix/simd_compat.h +++ b/include/pernix/simd_compat.h @@ -36,7 +36,7 @@ #elif defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) #include -#elif defined(__aarch64__) +#elif defined(__aarch64__) || defined(__arm64ec__) #include #endif From 50d14a6dcdd647bf46e6ed0bc37e03a57926c342 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Mon, 25 May 2026 18:37:03 +0200 Subject: [PATCH 08/43] WIP: implement NEON decompression functions --- tests/arm64/neon/.gitkeep => .clangd | 0 CMakeLists.txt | 12 ++ include/pernix/arm64/neon/common.h | 46 ++++- include/pernix/arm64/neon/decompression.h | 78 +++++++- include/pernix/arm64/neon/unpacking.h | 81 ++++++++- include/pernix/arm64/tables.h | 212 ++++++++++++++++++++++ src/CMakeLists.txt | 12 -- tests/CMakeLists.txt | 42 ++++- tests/arm64/neon/decompression_tests.cpp | 38 ++++ 9 files changed, 487 insertions(+), 34 deletions(-) rename tests/arm64/neon/.gitkeep => .clangd (100%) create mode 100644 include/pernix/arm64/tables.h create mode 100644 tests/arm64/neon/decompression_tests.cpp diff --git a/tests/arm64/neon/.gitkeep b/.clangd similarity index 100% rename from tests/arm64/neon/.gitkeep rename to .clangd diff --git a/CMakeLists.txt b/CMakeLists.txt index b8c8208..1cbead6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,18 @@ if (NOT PERNIX_ARCH_BACKEND IN_LIST PERNIX_VALID_ARCH_BACKENDS) message(FATAL_ERROR "Unsupported PERNIX_ARCH_BACKEND='${PERNIX_ARCH_BACKEND}'. Expected one of: ${PERNIX_VALID_ARCH_BACKENDS}") endif () +set(PERNIX_SELECTED_ARCH_BACKEND "${PERNIX_ARCH_BACKEND}") +if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "AUTO") + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|i[3-6]86|i686)$") + set(PERNIX_SELECTED_ARCH_BACKEND "X86") + elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") + set(PERNIX_SELECTED_ARCH_BACKEND "ARM64_NEON") + else () + set(PERNIX_SELECTED_ARCH_BACKEND "FALLBACK") + endif () +endif () +message(STATUS "Pernix architecture backend: ${PERNIX_SELECTED_ARCH_BACKEND}") + string(TOUPPER "${PERNIX_SIMDE_PROVIDER}" PERNIX_SIMDE_PROVIDER) set(PERNIX_VALID_SIMDE_PROVIDERS AUTO PACKAGE FETCH) if (NOT PERNIX_SIMDE_PROVIDER IN_LIST PERNIX_VALID_SIMDE_PROVIDERS) diff --git a/include/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h index 6efa777..f843908 100644 --- a/include/pernix/arm64/neon/common.h +++ b/include/pernix/arm64/neon/common.h @@ -11,7 +11,7 @@ static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t rem return tail_bytes; } -__always_inline int32x4x4_t neon_convert_int8x16_int32x4x2_t(const int8x16_t& input) { +__always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(const int8x16_t& input) { const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); @@ -23,6 +23,13 @@ __always_inline int32x4x4_t neon_convert_int8x16_int32x4x2_t(const int8x16_t& in }}; } +__always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(const int16x8_t& input) { + return {{ + vmovl_s16(vget_low_s16(input)), + vmovl_s16(vget_high_s16(input)), + }}; +} + __always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, const float32x4_t& scale) { return {{ vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), @@ -32,21 +39,32 @@ __always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, co }}; } -__always_inline uint8x16_t neon_load_tail_elements_int8(const uint8_t* input, const uint32_t tail_elements) { +__always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t& input, const float32x4_t& scale) { + return {{ + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + }}; +} + +__always_inline float32x4_t neon_dequantize_epi32(const int32x4_t& input, const float32x4_t& scale) { + return vmulq_f32(vcvtq_f32_s32(input), scale); +} + +__always_inline uint8x16_t neon_load_tail_elements_int8(const uint8_t* input, const uint32_t tail_bytes_count) { uint8_t buffer[16] = {0}; - std::memcpy(buffer, input, tail_elements * sizeof(uint8_t)); + std::memcpy(buffer, input, tail_bytes_count); return vld1q_u8(buffer); } -__always_inline uint16x8_t neon_load_tail_elements_int16(const uint8_t* input, const uint32_t tail_elements) { +__always_inline uint16x8_t neon_load_tail_elements_int16(const uint8_t* input, const uint32_t tail_bytes_count) { uint16_t buffer[8] = {0}; - std::memcpy(buffer, input, tail_elements * sizeof(uint16_t)); + std::memcpy(buffer, input, tail_bytes_count); return vld1q_u16(buffer); } -__always_inline uint32x4_t neon_load_tail_elements_int32(const uint8_t* input, const uint32_t tail_elements) { +__always_inline uint32x4_t neon_load_tail_elements_int32(const uint8_t* input, const uint32_t tail_bytes_count) { uint32_t buffer[4] = {0}; - std::memcpy(buffer, input, tail_elements * sizeof(uint32_t)); + std::memcpy(buffer, input, tail_bytes_count); return vld1q_u32(buffer); } @@ -94,6 +112,20 @@ __always_inline void neon_store_tail_elements_f32(float32_t* output, const float std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); } +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x2_t& data, const uint32_t tail_elements) { + float32_t buffer[8 * 2]; + for (uint32_t i = 0; i < 2; ++i) { + vst1q_f32(buffer + i * 4, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); +} + +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4_t& data, const uint32_t tail_elements) { + float32_t buffer[4]; + vst1q_f32(buffer, data); + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); +} + __always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x4_t& data, const uint32_t tail_elements) { float64_t buffer[8 * 4]; for (uint32_t i = 0; i < 4; ++i) { diff --git a/include/pernix/arm64/neon/decompression.h b/include/pernix/arm64/neon/decompression.h index 90ed78b..6c46f1a 100644 --- a/include/pernix/arm64/neon/decompression.h +++ b/include/pernix/arm64/neon/decompression.h @@ -25,7 +25,7 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input const uint8x16_t source = vld1q_u8(input); const int8x16_t unpacked = b128::neon_unpack_epi8_1to8(source); - const int32x4x4_t converted = neon_convert_int8x16_int32x4x2_t(unpacked); + const int32x4x4_t converted = neon_convert_int8x16_int32x4x4(unpacked); const float32x4x4_t dequantized = neon_dequantize_epi32(converted, scale_v); for (uint32_t j = 0; j < 4; ++j) { @@ -40,7 +40,7 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input const uint8x16_t tail_source = neon_load_tail_elements_int8(input, tail_bytes(BIT_WIDTH, remaining_elements)); const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8(tail_source); - const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x2_t(tail_unpacked); + const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x4(tail_unpacked); const float32x4x4_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); neon_store_tail_elements_f32(output, tail_dequantized, remaining_elements); @@ -58,13 +58,34 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu constexpr uint32_t iterations_8 = elements_per_block / 8; constexpr uint32_t remaining_elements = elements_per_block - iterations_8 * 8; + const float32x4_t scale_v = vdupq_n_f32(scale); + for (uint32_t i = 0; i < iterations_8; ++i) { - static_assert(true, "Not yet implemented"); + const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); + const int16x8_t unpacked = b128::neon_unpack_epi8_9to16(source); + + const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); + const float32x4x2_t dequantized = neon_dequantize_epi32(converted, scale_v); + + for (uint32_t j = 0; j < 2; ++j) { + vst1q_f32(output, dequantized.val[j]); + output += 4; + } + + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { - static_assert(true, "Not yet implemented"); + const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int16x8_t tail_unpacked = b128::neon_unpack_epi8_9to16(tail_source); + + const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); + const float32x4x2_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); + + neon_store_tail_elements_f32(output, tail_dequantized, remaining_elements); } + + return 0; } template @@ -76,13 +97,52 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp constexpr uint32_t iterations_4 = elements_per_block / 4; constexpr uint32_t remaining_elements = elements_per_block - iterations_4 * 4; + const float32x4_t scale_v = vdupq_n_f32(scale); + for (uint32_t i = 0; i < iterations_4; ++i) { - static_assert(true, "Not yet implemented"); + const uint32_t group_bit_start = i * 4u * BIT_WIDTH; + const uint8_t* group_input = input + group_bit_start / 8u; + const uint32x4_t source = vld1q_u32(reinterpret_cast(group_input)); + + int32x4_t unpacked; + if constexpr (BIT_WIDTH % 2 == 0) { + unpacked = b128::neon_unpack_epi8_17to24(source); + } else { + if (i % 2 == 0) { + unpacked = b128::neon_unpack_epi8_17to24(source); + } else { + unpacked = b128::neon_unpack_epi8_17to24(source); + } + } + + const float32x4_t dequantized = neon_dequantize_epi32(unpacked, scale_v); + + vst1q_f32(output, dequantized); + + output += 4; } if constexpr (remaining_elements > 0) { - static_assert(true, "Not yet implemented"); + constexpr uint32_t tail_bit_start = iterations_4 * 4u * BIT_WIDTH; + constexpr uint32_t tail_bit_offset = tail_bit_start % 8u; + const uint8_t* tail_input = input + tail_bit_start / 8u; + + constexpr uint32_t tail_bytes_count = (tail_bit_offset + remaining_elements * BIT_WIDTH + 7u) / 8u; + const uint32x4_t tail_source = neon_load_tail_elements_int32(tail_input, tail_bytes_count); + + int32x4_t tail_unpacked; + if constexpr (tail_bit_offset == 0) { + tail_unpacked = b128::neon_unpack_epi8_17to24(tail_source); + } else { + tail_unpacked = b128::neon_unpack_epi8_17to24(tail_source); + } + + const float32x4_t tail_dequantized = neon_dequantize_epi32(tail_unpacked, scale_v); + + neon_store_tail_elements_f32(output, tail_dequantized, remaining_elements); } + + return 0; } template @@ -101,6 +161,8 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input if constexpr (remaining_elements > 0) { static_assert(true, "Not yet implemented"); } + + return 0; } template @@ -119,6 +181,8 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu if constexpr (remaining_elements > 0) { static_assert(true, "Not yet implemented"); } + + return 0; } template @@ -137,6 +201,8 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp if constexpr (remaining_elements > 0) { static_assert(true, "Not yet implemented"); } + + return 0; } } // namespace internal diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h index c1ae78f..663eb9d 100644 --- a/include/pernix/arm64/neon/unpacking.h +++ b/include/pernix/arm64/neon/unpacking.h @@ -2,24 +2,91 @@ #define PERNIX_ARM64_NEON_UNPACKING_H #include +#include + +using namespace pernix::arm64::internal; namespace pernix::arm64::neon::internal::b128 { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline int8x16_t neon_unpack_epi8_1to8(const int8x16_t& input) { - return input; +__always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t& input) { + if constexpr (BIT_WIDTH == 8) { + return vreinterpretq_s8_u8(input); + } else { + using tables = table_unpacking; + + const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); + + uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); + + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + const uint8x16_t permuted2_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute2.data())); + + shifted = vorrq_u8(shifted, vshlq_u8(permuted2_u8, vld1q_s8(tables::shift2.data()))); + } + + constexpr int shift = 8 - BIT_WIDTH; + shifted = vshlq_n_u8(shifted, shift); + + if constexpr (SIGN_VALUES) { + return vshlq_s8(vreinterpretq_s8_u8(shifted), vdupq_n_s8(-shift)); + } else { + return vreinterpretq_s8_u8(vshlq_u8(shifted, vdupq_n_s8(-shift))); + } + } } template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline int16x8_t neon_unpack_epi8_9to16(const int16x8_t& input) { - return input; +__always_inline int16x8_t neon_unpack_epi8_9to16(const uint16x8_t& input) { + if constexpr (BIT_WIDTH == 16) { + return vreinterpretq_s16_u16(input); + } else { + using tables = table_unpacking; + + const uint8x16_t input_u8 = vreinterpretq_u8_u16(input); + + const uint8x16_t permuted1_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute1.data())); + + uint16x8_t shifted = vshlq_u16(vreinterpretq_u16_u8(permuted1_u8), vld1q_s16(tables::shift1.data())); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const uint8x16_t permuted2_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute2.data())); + + const uint16x8_t shifted2 = vshlq_u16(vreinterpretq_u16_u8(permuted2_u8), vld1q_s16(tables::shift2.data())); + + shifted = vorrq_u16(shifted, shifted2); + } + + constexpr int shift = 16 - BIT_WIDTH; + shifted = vshlq_n_u16(shifted, shift); + + if constexpr (SIGN_VALUES) { + return vshlq_s16(vreinterpretq_s16_u16(shifted), vdupq_n_s16(-shift)); + } else { + return vreinterpretq_s16_u16(vshlq_u16(shifted, vdupq_n_s16(-shift))); + } + } } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline int32x4_t neon_unpack_epi8_17to24(const int32x4_t& input) { - return input; +__always_inline int32x4_t neon_unpack_epi8_17to24(const uint32x4_t& input) { + using tables = table_unpacking; + + const uint8x16_t input_8 = vreinterpretq_u8_u32(input); + + const uint8x16_t permuted_u8 = vqtbl1q_u8(input_8, vld1q_u8(tables::permute.data())); + + const uint32x4_t value = vshlq_u32(vreinterpretq_u32_u8(permuted_u8), vld1q_s32(tables::shift.data())); + + if constexpr (SIGN_VALUES) { + constexpr int sign_shift = 32 - BIT_WIDTH; + return vshrq_n_s32(vreinterpretq_s32_u32(vshlq_n_u32(value, sign_shift)), sign_shift); + } else { + constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1u; + return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); + } } } // namespace pernix::arm64::neon::internal::b128 diff --git a/include/pernix/arm64/tables.h b/include/pernix/arm64/tables.h new file mode 100644 index 0000000..233bbc4 --- /dev/null +++ b/include/pernix/arm64/tables.h @@ -0,0 +1,212 @@ +#ifndef PERNIX_ARM64_TABLES_H +#define PERNIX_ARM64_TABLES_H + +#include +#include +#include + +namespace pernix::arm64::internal { +namespace detail { +inline constexpr std::size_t neon_vector_width = 128; +inline constexpr uint8_t inactive_lane = 0xff; + +template +constexpr bool table_indices_are_valid(const std::array& table) { + for (const uint8_t index : table) { + if (index != inactive_lane && index >= Elements) { + return false; + } + } + + return true; +} + +template +constexpr std::array make_primary_permute() { + static_assert(LANE_BITS % 8 == 0); + + constexpr std::size_t lane_bytes = LANE_BITS / 8; + static_assert(ELEMENTS % lane_bytes == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t first_byte = bit_start / 8; + const std::size_t base = entry * lane_bytes; + + for (std::size_t lane_byte = 0; lane_byte < lane_bytes; ++lane_byte) { + table[base + lane_byte] = static_cast(first_byte + lane_byte); + } + } + + return table; +} + +template +constexpr std::array make_spill_permute() { + static_assert(LANE_BITS % 8 == 0); + + constexpr std::size_t lane_bytes = LANE_BITS / 8; + static_assert(ELEMENTS % lane_bytes == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t first_byte = bit_start / 8; + const std::size_t bit_offset = bit_start % 8; + const std::size_t base = entry * lane_bytes; + + if (bit_offset + BIT_WIDTH > LANE_BITS) { + table[base] = static_cast(first_byte + lane_bytes); + } + } + + return table; +} + +template +constexpr std::array make_shift_right() { + std::array table{}; + table.fill(0); + + for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t bit_offset = bit_start % 8u; + + table[entry] = -static_cast(bit_offset); + } + + return table; +} + +template +constexpr std::array make_shift_left_for_spill() { + std::array table{}; + table.fill(0); + + for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t bit_offset = bit_start % 8u; + const bool spills = bit_offset + BIT_WIDTH > LANE_BITS; + + table[entry] = spills ? static_cast(LANE_BITS - bit_offset) : 0; + } + + return table; +} + +template +constexpr std::array make_contiguous_permute_32() { + static_assert(ELEMENTS % 4 == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (std::size_t entry = 0; entry < ELEMENTS / 4; ++entry) { + const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + const std::size_t bit_end = bit_start + BIT_WIDTH - 1; + const std::size_t first_byte = bit_start / 8; + const std::size_t last_byte = bit_end / 8; + const std::size_t base = entry * 4; + + for (std::size_t byte = first_byte; byte <= last_byte; ++byte) { + table[base + (byte - first_byte)] = static_cast(byte); + } + } + + return table; +} + +template +constexpr std::array make_shift_right_32() { + std::array table{}; + table.fill(0); + + for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { + const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + + table[entry] = -static_cast(bit_start % 8u); + } + + return table; +} +} // namespace detail + +template +struct table_unpacking; + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && VECTOR_WIDTH == detail::neon_vector_width) +struct table_unpacking { +private: + static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 8; + +public: + static constexpr uint8_t bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute1 = + detail::make_primary_permute(); + alignas(64) static constexpr std::array permute2 = + detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); + alignas(64) static constexpr std::array shift2 = + detail::make_shift_left_for_spill(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 16); + static_assert(detail::table_indices_are_valid(permute1)); + static_assert(detail::table_indices_are_valid(permute2)); +}; + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && VECTOR_WIDTH == detail::neon_vector_width) +struct table_unpacking { +private: + static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 16; + +public: + static constexpr uint8_t bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute1 = + detail::make_primary_permute(); + alignas(64) static constexpr std::array permute2 = + detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = + detail::make_shift_right(); + alignas(64) static constexpr std::array shift2 = + detail::make_shift_left_for_spill(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 8); + static_assert(detail::table_indices_are_valid(permute1)); + static_assert(detail::table_indices_are_valid(permute2)); +}; + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && VECTOR_WIDTH == detail::neon_vector_width && START_BIT_OFFSET < 8) +struct table_unpacking { +private: + static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 32; + +public: + static constexpr uint8_t bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute = + detail::make_contiguous_permute_32(); + alignas(64) static constexpr std::array shift = + detail::make_shift_right_32(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 4); + static_assert(detail::table_indices_are_valid(permute)); +}; +} // namespace pernix::arm64::internal + +#endif // PERNIX_ARM64_TABLES_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8c79ab6..57d2f0a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,18 +11,6 @@ file(GLOB_RECURSE set(PERNIX_SOURCES ${PERNIX_COMMON_SOURCES}) -set(PERNIX_SELECTED_ARCH_BACKEND "${PERNIX_ARCH_BACKEND}") -if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "AUTO") - if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|i[3-6]86|i686)$") - set(PERNIX_SELECTED_ARCH_BACKEND "X86") - elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") - set(PERNIX_SELECTED_ARCH_BACKEND "ARM64_NEON") - else () - set(PERNIX_SELECTED_ARCH_BACKEND "FALLBACK") - endif () -endif () -message(STATUS "Pernix architecture backend: ${PERNIX_SELECTED_ARCH_BACKEND}") - if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "X86") file(GLOB_RECURSE PERNIX_X86_SOURCES diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 370ec3b..b5ffe57 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,12 +2,50 @@ find_package(PkgConfig) pkg_search_module(GTEST REQUIRED gtest) include(CheckCXXCompilerFlag) +file(GLOB + PERNIX_ROOT_TEST_SOURCES + CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp +) + file(GLOB_RECURSE - SOURCE_FILES + PERNIX_FALLBACK_TEST_SOURCES CONFIGURE_DEPENDS - *.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/fallback/*.cpp ) +set(SOURCE_FILES ${PERNIX_ROOT_TEST_SOURCES} ${PERNIX_FALLBACK_TEST_SOURCES}) + +if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "X86") + file(GLOB_RECURSE + PERNIX_X86_TEST_SOURCES + CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/x86/*.cpp + ) + list(APPEND SOURCE_FILES ${PERNIX_X86_TEST_SOURCES}) +elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_NEON") + file(GLOB_RECURSE + PERNIX_ARM64_NEON_TEST_SOURCES + CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/arm64/neon/*.cpp + ) + list(APPEND SOURCE_FILES ${PERNIX_ARM64_NEON_TEST_SOURCES}) +elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE") + file(GLOB_RECURSE + PERNIX_ARM64_SVE_TEST_SOURCES + CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/arm64/sve/*.cpp + ) + list(APPEND SOURCE_FILES ${PERNIX_ARM64_SVE_TEST_SOURCES}) +elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE2") + file(GLOB_RECURSE + PERNIX_ARM64_SVE2_TEST_SOURCES + CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/arm64/sve2/*.cpp + ) + list(APPEND SOURCE_FILES ${PERNIX_ARM64_SVE2_TEST_SOURCES}) +endif () + file(GLOB_RECURSE HEADER_FILES CONFIGURE_DEPENDS diff --git a/tests/arm64/neon/decompression_tests.cpp b/tests/arm64/neon/decompression_tests.cpp new file mode 100644 index 0000000..69229be --- /dev/null +++ b/tests/arm64/neon/decompression_tests.cpp @@ -0,0 +1,38 @@ +#include +#include + +#ifdef PERNIX_BACKEND_ARM64_NEON + +using namespace pernix::arm64::neon; + +TYPED_TEST(DecompressionTest, NeonDecompressBlock) { + std::vector > decompressedData(this->testSet.numberOfBlocks); + + for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { + decompressedData[block].resize(this->testSet.elementsPerBlock); + + neon_decompress_block( + this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); + } + + for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { + expectDecompressedBlockNearSource(*this, decompressedData[block], block); + } +} + +TYPED_TEST(DecompressionTest64, NeonDecompressBlock) { + std::vector > decompressedData(this->testSet.numberOfBlocks); + + for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { + decompressedData[block].resize(this->testSet.elementsPerBlock); + + neon_decompress_block( + this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); + } + + for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { + expectDecompressedBlockNearSource(*this, decompressedData[block], block); + } +} + +#endif \ No newline at end of file From 9eca7046bce1aa7be90f45420e698a4984f62af2 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Tue, 26 May 2026 00:04:55 +0200 Subject: [PATCH 09/43] WIP: implement NEON decompression functions --- include/pernix/arm64/neon/common.h | 68 ++++++++- include/pernix/arm64/neon/decompression.h | 162 +++++++++++++++------- include/pernix/arm64/neon/unpacking.h | 11 +- include/pernix/arm64/tables.h | 4 +- 4 files changed, 189 insertions(+), 56 deletions(-) diff --git a/include/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h index f843908..8e517fa 100644 --- a/include/pernix/arm64/neon/common.h +++ b/include/pernix/arm64/neon/common.h @@ -2,9 +2,14 @@ #define PERNIX_ARM64_NEON_COMMON_H #include + #include namespace pernix::arm64::neon::internal { +struct float64x2x8_t { + float64x2_t val[8]; +}; + static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t remaining_elements) { const uint32_t tail_bits = remaining_elements * bit_width; const uint32_t tail_bytes = (tail_bits + 7u) / 8u; @@ -50,6 +55,47 @@ __always_inline float32x4_t neon_dequantize_epi32(const int32x4_t& input, const return vmulq_f32(vcvtq_f32_s32(input), scale); } +__always_inline float64x2_t neon_dequantize_epi32_f64(const int32x2_t& input, const float64x2_t& scale) { + return vmulq_f64(vcvtq_f64_s64(vmovl_s32(input)), scale); +} + +__always_inline float64x2x2_t neon_dequantize_epi32_f64(const int32x4_t& input, const float64x2_t& scale) { + return {{ + neon_dequantize_epi32_f64(vget_low_s32(input), scale), + neon_dequantize_epi32_f64(vget_high_s32(input), scale), + }}; +} + +__always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t& input, const float64x2_t& scale) { + const float64x2x2_t dequantized_low = neon_dequantize_epi32_f64(input.val[0], scale); + const float64x2x2_t dequantized_high = neon_dequantize_epi32_f64(input.val[1], scale); + + return {{ + dequantized_low.val[0], + dequantized_low.val[1], + dequantized_high.val[0], + dequantized_high.val[1], + }}; +} + +__always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input, const float64x2_t& scale) { + const float64x2x2_t dequantized0 = neon_dequantize_epi32_f64(input.val[0], scale); + const float64x2x2_t dequantized1 = neon_dequantize_epi32_f64(input.val[1], scale); + const float64x2x2_t dequantized2 = neon_dequantize_epi32_f64(input.val[2], scale); + const float64x2x2_t dequantized3 = neon_dequantize_epi32_f64(input.val[3], scale); + + return {{ + dequantized0.val[0], + dequantized0.val[1], + dequantized1.val[0], + dequantized1.val[1], + dequantized2.val[0], + dequantized2.val[1], + dequantized3.val[0], + dequantized3.val[1], + }}; +} + __always_inline uint8x16_t neon_load_tail_elements_int8(const uint8_t* input, const uint32_t tail_bytes_count) { uint8_t buffer[16] = {0}; std::memcpy(buffer, input, tail_bytes_count); @@ -127,12 +173,28 @@ __always_inline void neon_store_tail_elements_f32(float32_t* output, const float } __always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x4_t& data, const uint32_t tail_elements) { - float64_t buffer[8 * 4]; + float64_t buffer[2 * 4]; for (uint32_t i = 0; i < 4; ++i) { vst1q_f64(buffer + i * 2, data.val[i]); } std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); } -} // namespace pernix::arm64::neon::internal -#endif //PERNIX_ARM64_NEON_COMMON_H \ No newline at end of file +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x2_t& data, const uint32_t tail_elements) { + float64_t buffer[2 * 2]; + for (uint32_t i = 0; i < 2; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); +} + +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x8_t& data, const uint32_t tail_elements) { + float64_t buffer[2 * 8]; + for (uint32_t i = 0; i < 8; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); +} +} // namespace pernix::arm64::neon::internal + +#endif // PERNIX_ARM64_NEON_COMMON_H diff --git a/include/pernix/arm64/neon/decompression.h b/include/pernix/arm64/neon/decompression.h index 6c46f1a..cfc051e 100644 --- a/include/pernix/arm64/neon/decompression.h +++ b/include/pernix/arm64/neon/decompression.h @@ -1,9 +1,9 @@ #ifndef PERNIX_ARM64_NEON_DECOMPRESSION_H #define PERNIX_ARM64_NEON_DECOMPRESSION_H -#include -#include #include +#include +#include #include #include @@ -12,8 +12,7 @@ namespace pernix::arm64::neon { namespace internal { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_16 = elements_per_block / 16; @@ -51,8 +50,7 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; @@ -90,8 +88,7 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_4 = elements_per_block / 4; @@ -123,12 +120,12 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp } if constexpr (remaining_elements > 0) { - constexpr uint32_t tail_bit_start = iterations_4 * 4u * BIT_WIDTH; + constexpr uint32_t tail_bit_start = iterations_4 * 4u * BIT_WIDTH; constexpr uint32_t tail_bit_offset = tail_bit_start % 8u; - const uint8_t* tail_input = input + tail_bit_start / 8u; + const uint8_t* tail_input = input + tail_bit_start / 8u; constexpr uint32_t tail_bytes_count = (tail_bit_offset + remaining_elements * BIT_WIDTH + 7u) / 8u; - const uint32x4_t tail_source = neon_load_tail_elements_int32(tail_input, tail_bytes_count); + const uint32x4_t tail_source = neon_load_tail_elements_int32(tail_input, tail_bytes_count); int32x4_t tail_unpacked; if constexpr (tail_bit_offset == 0) { @@ -147,19 +144,37 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_8 * 8; + constexpr uint32_t iterations_16 = elements_per_block / 16; + constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16; - for (uint32_t i = 0; i < iterations_8; ++i) { - static_assert(true, "Not yet implemented"); + const float64x2_t scale_v = vdupq_n_f64(scale); + + for (uint32_t i = 0; i < iterations_16; ++i) { + const uint8x16_t source = vld1q_u8(input); + const int8x16_t unpacked = b128::neon_unpack_epi8_1to8(source); + + const int32x4x4_t converted = neon_convert_int8x16_int32x4x4(unpacked); + const float64x2x8_t dequantized = neon_dequantize_epi32_f64(converted, scale_v); + + for (uint32_t j = 0; j < 8; ++j) { + vst1q_f64(output, dequantized.val[j]); + output += 2; + } + + input += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { - static_assert(true, "Not yet implemented"); + const uint8x16_t tail_source = neon_load_tail_elements_int8(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8(tail_source); + + const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x4(tail_unpacked); + const float64x2x8_t tail_dequantized = neon_dequantize_epi32_f64(tail_converted, scale_v); + + neon_store_tail_elements_f64(output, tail_dequantized, remaining_elements); } return 0; @@ -167,19 +182,37 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_4 = elements_per_block / 4; - constexpr uint32_t remaining_elements = elements_per_block - iterations_4 * 4; + constexpr uint32_t iterations_8 = elements_per_block / 8; + constexpr uint32_t remaining_elements = elements_per_block - iterations_8 * 8; - for (uint32_t i = 0; i < iterations_4; ++i) { - static_assert(true, "Not yet implemented"); + const float64x2_t scale_v = vdupq_n_f64(scale); + + for (uint32_t i = 0; i < iterations_8; ++i) { + const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); + const int16x8_t unpacked = b128::neon_unpack_epi8_9to16(source); + + const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); + const float64x2x4_t dequantized = neon_dequantize_epi32_f64(converted, scale_v); + + for (uint32_t j = 0; j < 4; ++j) { + vst1q_f64(output, dequantized.val[j]); + output += 2; + } + + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { - static_assert(true, "Not yet implemented"); + const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int16x8_t tail_unpacked = b128::neon_unpack_epi8_9to16(tail_source); + + const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); + const float64x2x4_t tail_dequantized = neon_dequantize_epi32_f64(tail_converted, scale_v); + + neon_store_tail_elements_f64(output, tail_dequantized, remaining_elements); } return 0; @@ -187,29 +220,65 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_2 = elements_per_block / 2; - constexpr uint32_t remaining_elements = elements_per_block - iterations_2 * 2; + constexpr uint32_t iterations_4 = elements_per_block / 4; + constexpr uint32_t remaining_elements = elements_per_block - iterations_4 * 4; + + const float64x2_t scale_v = vdupq_n_f64(scale); + + for (uint32_t i = 0; i < iterations_4; ++i) { + const uint32_t group_bit_start = i * 4u * BIT_WIDTH; + const uint8_t* group_input = input + group_bit_start / 8u; + const uint32x4_t source = vld1q_u32(reinterpret_cast(group_input)); + + int32x4_t unpacked; + if constexpr (BIT_WIDTH % 2 == 0) { + unpacked = b128::neon_unpack_epi8_17to24(source); + } else { + if (i % 2 == 0) { + unpacked = b128::neon_unpack_epi8_17to24(source); + } else { + unpacked = b128::neon_unpack_epi8_17to24(source); + } + } + + const float64x2x2_t dequantized = neon_dequantize_epi32_f64(unpacked, scale_v); - for (uint32_t i = 0; i < iterations_2; ++i) { - static_assert(true, "Not yet implemented"); + for (uint32_t j = 0; j < 2; ++j) { + vst1q_f64(output, dequantized.val[j]); + output += 2; + } } if constexpr (remaining_elements > 0) { - static_assert(true, "Not yet implemented"); + constexpr uint32_t tail_bit_start = iterations_4 * 4u * BIT_WIDTH; + constexpr uint32_t tail_bit_offset = tail_bit_start % 8u; + const uint8_t* tail_input = input + tail_bit_start / 8u; + + constexpr uint32_t tail_bytes_count = (tail_bit_offset + remaining_elements * BIT_WIDTH + 7u) / 8u; + const uint32x4_t tail_source = neon_load_tail_elements_int32(tail_input, tail_bytes_count); + + int32x4_t tail_unpacked; + if constexpr (tail_bit_offset == 0) { + tail_unpacked = b128::neon_unpack_epi8_17to24(tail_source); + } else { + tail_unpacked = b128::neon_unpack_epi8_17to24(tail_source); + } + + const float64x2x2_t tail_dequantized = neon_dequantize_epi32_f64(tail_unpacked, scale_v); + + neon_store_tail_elements_f64(output, tail_dequantized, remaining_elements); } return 0; } -} // namespace internal +} // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int neon_decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return internal::neon_decompress_block_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { @@ -222,8 +291,7 @@ __always_inline int neon_decompress_block(const uint8_t* __restrict__ input, con template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int neon_decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return internal::neon_decompress_block_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { @@ -236,14 +304,13 @@ __always_inline int neon_decompress_block(const uint8_t* __restrict__ input, con template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { +int neon_decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { const uint8_t* block_input = input; float_t* block_output = output; for (uint32_t block = 0; block < blocks; ++block) { neon_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -252,14 +319,13 @@ int neon_decompress_blocks(const uint8_t* __restrict__ input, const float_t scal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { +int neon_decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { const uint8_t* block_input = input; double_t* block_output = output; for (uint32_t block = 0; block < blocks; ++block) { neon_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; @@ -271,19 +337,17 @@ extern "C" { int neon_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -int neon_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output); +int neon_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); int neon_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); -int neon_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output, uint32_t blocks); +int neon_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, + uint32_t blocks); #ifdef __cplusplus } #endif -} // namespace pernix::arm64::neon +} // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_DECOMPRESSION_H diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h index 663eb9d..b1f8585 100644 --- a/include/pernix/arm64/neon/unpacking.h +++ b/include/pernix/arm64/neon/unpacking.h @@ -1,8 +1,8 @@ #ifndef PERNIX_ARM64_NEON_UNPACKING_H #define PERNIX_ARM64_NEON_UNPACKING_H -#include #include +#include using namespace pernix::arm64::internal; @@ -12,6 +12,13 @@ template __always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t& input) { if constexpr (BIT_WIDTH == 8) { return vreinterpretq_s8_u8(input); + } else if constexpr (BIT_WIDTH == 1) { + using tables = table_unpacking; + + const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); + const uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); + + return vreinterpretq_s8_u8(vandq_u8(shifted, vdupq_n_u8(1))); } else { using tables = table_unpacking; @@ -88,6 +95,6 @@ __always_inline int32x4_t neon_unpack_epi8_17to24(const uint32x4_t& input) { return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); } } -} // namespace pernix::arm64::neon::internal::b128 +} // namespace pernix::arm64::neon::internal::b128 #endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/arm64/tables.h b/include/pernix/arm64/tables.h index 233bbc4..60e1dfe 100644 --- a/include/pernix/arm64/tables.h +++ b/include/pernix/arm64/tables.h @@ -134,7 +134,7 @@ constexpr std::array make_shift_right_32() { return table; } -} // namespace detail +} // namespace detail template struct table_unpacking; @@ -207,6 +207,6 @@ struct table_unpacking { static_assert(SHIFT_ELEMENTS == 4); static_assert(detail::table_indices_are_valid(permute)); }; -} // namespace pernix::arm64::internal +} // namespace pernix::arm64::internal #endif // PERNIX_ARM64_TABLES_H From af90ba8ac32167d689c220feb0dca14314affdf8 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 28 May 2026 13:22:46 +0200 Subject: [PATCH 10/43] WIP: Implement SVE2 decompression functions and update related headers for ARM64 --- include/pernix/arm64/neon/decompression.h | 36 +-- include/pernix/arm64/{ => neon}/tables.h | 26 +- include/pernix/arm64/neon/unpacking.h | 10 +- include/pernix/arm64/sve2/decompression.h | 346 ++++++++++++++++++++-- include/pernix/arm64/sve2/tables.h | 143 +++++++++ include/pernix/arm64/sve2/unpacking.h | 87 +++++- include/pernix/simd_compat.h | 5 + src/arm64/neon/decompression.cpp | 140 ++++++++- src/arm64/sve2/decompression.cpp | 76 ++++- tests/arm64/sve2/.gitkeep | 0 tests/arm64/sve2/decompression_tests.cpp | 38 +++ 11 files changed, 833 insertions(+), 74 deletions(-) rename include/pernix/arm64/{ => neon}/tables.h (94%) create mode 100644 include/pernix/arm64/sve2/tables.h delete mode 100644 tests/arm64/sve2/.gitkeep create mode 100644 tests/arm64/sve2/decompression_tests.cpp diff --git a/include/pernix/arm64/neon/decompression.h b/include/pernix/arm64/neon/decompression.h index cfc051e..583948f 100644 --- a/include/pernix/arm64/neon/decompression.h +++ b/include/pernix/arm64/neon/decompression.h @@ -60,7 +60,7 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu for (uint32_t i = 0; i < iterations_8; ++i) { const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); - const int16x8_t unpacked = b128::neon_unpack_epi8_9to16(source); + const int16x8_t unpacked = b128::neon_unpack_epi16_9to16(source); const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); const float32x4x2_t dequantized = neon_dequantize_epi32(converted, scale_v); @@ -75,7 +75,7 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu if constexpr (remaining_elements > 0) { const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int16x8_t tail_unpacked = b128::neon_unpack_epi8_9to16(tail_source); + const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16(tail_source); const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); const float32x4x2_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); @@ -103,12 +103,12 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp int32x4_t unpacked; if constexpr (BIT_WIDTH % 2 == 0) { - unpacked = b128::neon_unpack_epi8_17to24(source); + unpacked = b128::neon_unpack_epi32_17to24(source); } else { if (i % 2 == 0) { - unpacked = b128::neon_unpack_epi8_17to24(source); + unpacked = b128::neon_unpack_epi32_17to24(source); } else { - unpacked = b128::neon_unpack_epi8_17to24(source); + unpacked = b128::neon_unpack_epi32_17to24(source); } } @@ -129,9 +129,9 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp int32x4_t tail_unpacked; if constexpr (tail_bit_offset == 0) { - tail_unpacked = b128::neon_unpack_epi8_17to24(tail_source); + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } else { - tail_unpacked = b128::neon_unpack_epi8_17to24(tail_source); + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } const float32x4_t tail_dequantized = neon_dequantize_epi32(tail_unpacked, scale_v); @@ -192,7 +192,7 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu for (uint32_t i = 0; i < iterations_8; ++i) { const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); - const int16x8_t unpacked = b128::neon_unpack_epi8_9to16(source); + const int16x8_t unpacked = b128::neon_unpack_epi16_9to16(source); const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); const float64x2x4_t dequantized = neon_dequantize_epi32_f64(converted, scale_v); @@ -207,7 +207,7 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu if constexpr (remaining_elements > 0) { const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int16x8_t tail_unpacked = b128::neon_unpack_epi8_9to16(tail_source); + const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16(tail_source); const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); const float64x2x4_t tail_dequantized = neon_dequantize_epi32_f64(tail_converted, scale_v); @@ -235,12 +235,12 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp int32x4_t unpacked; if constexpr (BIT_WIDTH % 2 == 0) { - unpacked = b128::neon_unpack_epi8_17to24(source); + unpacked = b128::neon_unpack_epi32_17to24(source); } else { if (i % 2 == 0) { - unpacked = b128::neon_unpack_epi8_17to24(source); + unpacked = b128::neon_unpack_epi32_17to24(source); } else { - unpacked = b128::neon_unpack_epi8_17to24(source); + unpacked = b128::neon_unpack_epi32_17to24(source); } } @@ -262,9 +262,9 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp int32x4_t tail_unpacked; if constexpr (tail_bit_offset == 0) { - tail_unpacked = b128::neon_unpack_epi8_17to24(tail_source); + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } else { - tail_unpacked = b128::neon_unpack_epi8_17to24(tail_source); + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } const float64x2x2_t tail_dequantized = neon_dequantize_epi32_f64(tail_unpacked, scale_v); @@ -274,7 +274,7 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp return 0; } -} // namespace internal +} // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) @@ -310,7 +310,7 @@ int neon_decompress_blocks(const uint8_t* __restrict__ input, const float_t scal for (uint32_t block = 0; block < blocks; ++block) { neon_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -325,7 +325,7 @@ int neon_decompress_blocks(const uint8_t* __restrict__ input, const double_t sca for (uint32_t block = 0; block < blocks; ++block) { neon_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; @@ -348,6 +348,6 @@ int neon_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ in #ifdef __cplusplus } #endif -} // namespace pernix::arm64::neon +} // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_DECOMPRESSION_H diff --git a/include/pernix/arm64/tables.h b/include/pernix/arm64/neon/tables.h similarity index 94% rename from include/pernix/arm64/tables.h rename to include/pernix/arm64/neon/tables.h index 60e1dfe..d085551 100644 --- a/include/pernix/arm64/tables.h +++ b/include/pernix/arm64/neon/tables.h @@ -1,24 +1,21 @@ -#ifndef PERNIX_ARM64_TABLES_H -#define PERNIX_ARM64_TABLES_H +#ifndef PERNIX_ARM64_NEON_TABLES_H +#define PERNIX_ARM64_NEON_TABLES_H +#include #include #include #include -namespace pernix::arm64::internal { +namespace pernix::arm64::neon::internal { namespace detail { inline constexpr std::size_t neon_vector_width = 128; inline constexpr uint8_t inactive_lane = 0xff; template constexpr bool table_indices_are_valid(const std::array& table) { - for (const uint8_t index : table) { - if (index != inactive_lane && index >= Elements) { - return false; - } - } - - return true; + return std::ranges::all_of(table, [](const uint8_t index) { + return index == inactive_lane || index < Elements; + }); } template @@ -134,7 +131,7 @@ constexpr std::array make_shift_right_32() { return table; } -} // namespace detail +} // namespace detail template struct table_unpacking; @@ -207,6 +204,9 @@ struct table_unpacking { static_assert(SHIFT_ELEMENTS == 4); static_assert(detail::table_indices_are_valid(permute)); }; -} // namespace pernix::arm64::internal -#endif // PERNIX_ARM64_TABLES_H +template +struct table_packing; +} // namespace pernix::arm64::internal + +#endif // PERNIX_ARM64_NEON_TABLES_H diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h index b1f8585..6ac0e20 100644 --- a/include/pernix/arm64/neon/unpacking.h +++ b/include/pernix/arm64/neon/unpacking.h @@ -1,10 +1,10 @@ #ifndef PERNIX_ARM64_NEON_UNPACKING_H #define PERNIX_ARM64_NEON_UNPACKING_H -#include +#include #include -using namespace pernix::arm64::internal; +using namespace pernix::arm64::neon::internal; namespace pernix::arm64::neon::internal::b128 { template @@ -45,7 +45,7 @@ __always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t& input) { template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline int16x8_t neon_unpack_epi8_9to16(const uint16x8_t& input) { +__always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t& input) { if constexpr (BIT_WIDTH == 16) { return vreinterpretq_s16_u16(input); } else { @@ -78,7 +78,7 @@ __always_inline int16x8_t neon_unpack_epi8_9to16(const uint16x8_t& input) { template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline int32x4_t neon_unpack_epi8_17to24(const uint32x4_t& input) { +__always_inline int32x4_t neon_unpack_epi32_17to24(const uint32x4_t& input) { using tables = table_unpacking; const uint8x16_t input_8 = vreinterpretq_u8_u32(input); @@ -95,6 +95,6 @@ __always_inline int32x4_t neon_unpack_epi8_17to24(const uint32x4_t& input) { return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); } } -} // namespace pernix::arm64::neon::internal::b128 +} // namespace pernix::arm64::neon::internal::b128 #endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/arm64/sve2/decompression.h b/include/pernix/arm64/sve2/decompression.h index c27f08a..198a8fc 100644 --- a/include/pernix/arm64/sve2/decompression.h +++ b/include/pernix/arm64/sve2/decompression.h @@ -1,43 +1,352 @@ #ifndef PERNIX_ARM64_SVE2_DECOMPRESSION_H #define PERNIX_ARM64_SVE2_DECOMPRESSION_H +#include +#include #include +#include #include #include +#include -namespace pernix { +namespace pernix::arm64::sve2 { namespace internal { -template -inline constexpr bool sve2_decompression_unimplemented_v = false; -} // namespace internal +template +[[nodiscard]] __always_inline constexpr uint32_t packed_bytes(const uint32_t elements) { + return (elements * BIT_WIDTH + 7) / 8; +} + +[[nodiscard]] __always_inline svuint8_t sve2_load_packed_bytes(const uint8_t* __restrict__ input, const uint32_t bytes) { + const svbool_t pg = svwhilelt_b8(uint64_t{0}, static_cast(bytes)); + return svld1_u8(pg, input); +} + +template +__always_inline void sve2_store_dequantized_i8_f32(svint8_t values, const svfloat32_t scale_v, float_t* __restrict__ output, + const uint32_t count) { + alignas(64) std::vector temp(svcntb()); + + svst1_s8(svptrue_b8(), temp.data(), values); + + uint32_t offset = 0; + while (offset < count) { + const svbool_t pg = svwhilelt_b32(static_cast(offset), static_cast(count)); + + svfloat32_t dequantized; + + if constexpr (SIGN_VALUES) { + const svint32_t widened = svld1sb_s32(pg, temp.data() + offset); + dequantized = svmul_f32_x(pg, svcvt_f32_s32_x(pg, widened), scale_v); + } else { + const svuint32_t widened = svld1ub_u32(pg, reinterpret_cast(temp.data() + offset)); + dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, widened), scale_v); + } + + svst1_f32(pg, output + offset, dequantized); + + offset += static_cast(svcntw()); + } +} + +template +__always_inline void sve2_store_dequantized_i8_f64(svint8_t values, const double_t scale, double_t* __restrict__ output, + const uint32_t count) { + std::vector temp(svcntb()); + + svst1_s8(svptrue_b8(), temp.data(), values); + + for (uint32_t i = 0; i < count; ++i) { + if constexpr (SIGN_VALUES) { + output[i] = static_cast(temp[i]) * scale; + } else { + output[i] = static_cast(static_cast(temp[i])) * scale; + } + } +} + +template +__always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfloat32_t scale_v, float_t* __restrict__ output, + const uint32_t count) { + alignas(64) std::vector temp(svcnth()); + + svst1_s16(svptrue_b16(), temp.data(), values); + + uint32_t offset = 0; + while (offset < count) { + const svbool_t pg = svwhilelt_b32(static_cast(offset), static_cast(count)); + + svfloat32_t dequantized; + if constexpr (SIGN_VALUES) { + const svint32_t widened = svld1sh_s32(pg, temp.data() + offset); + dequantized = svmul_f32_x(pg, svcvt_f32_s32_x(pg, widened), scale_v); + } else { + const svuint32_t widened = svld1uh_u32(pg, reinterpret_cast(temp.data() + offset)); + dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, widened), scale_v); + } + + svst1_f32(pg, output + offset, dequantized); + + offset += static_cast(svcntw()); + } +} + +template +__always_inline void sve2_store_dequantized_i16_f64(svint16_t values, const double_t scale, double_t* __restrict__ output, + const uint32_t count) { + std::vector temp(svcnth()); + + svst1_s16(svptrue_b16(), temp.data(), values); + + for (uint32_t i = 0; i < count; ++i) { + if constexpr (SIGN_VALUES) { + output[i] = static_cast(temp[i]) * scale; + } else { + output[i] = static_cast(static_cast(temp[i])) * scale; + } + } +} + +template +__always_inline void sve2_store_dequantized_i32_f32(svint32_t values, const svfloat32_t scale_v, float_t* __restrict__ output, + const uint32_t count) { + const svbool_t pg = svwhilelt_b32(uint64_t{0}, static_cast(count)); + + svfloat32_t dequantized; + if constexpr (SIGN_VALUES) { + dequantized = svmul_f32_x(pg, svcvt_f32_s32_x(pg, values), scale_v); + } else { + dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, svreinterpret_u32_s32(values)), scale_v); + } + + svst1_f32(pg, output, dequantized); +} template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_block(const uint8_t*, float_t, float_t*) { - static_assert(internal::sve2_decompression_unimplemented_v, "ARM64 SVE2 decompression is not implemented yet"); + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcntb()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const svfloat32_t scale_v = svdup_n_f32(scale); + + const table_unpacking table; + const svuint8_t permute = table.permute(); + const svuint8_t shift = table.shift(); + svuint8_t spill_permute = svdup_n_u8(0); + svuint8_t spill_shift = svdup_n_u8(0); + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + spill_permute = table.spill_permute(); + spill_shift = table.spill_shift(); + } + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint32_t bytes = packed_bytes(count); + const uint8_t* chunk_input = input + input_bit_offset / 8; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svint8_t unpacked = sve2_unpack_epi8_1to8(source, permute, shift, spill_permute, spill_shift); + + sve2_store_dequantized_i8_f32(unpacked, scale_v, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcnth()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const svfloat32_t scale_v = svdup_n_f32(scale); + + const table_unpacking table; + const svuint8_t permute = table.permute(); + const svuint16_t shift = table.shift(); + svuint8_t spill_permute = svdup_n_u8(0); + svuint16_t spill_shift = svdup_n_u16(0); + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + spill_permute = table.spill_permute(); + spill_shift = table.spill_shift(); + } + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint32_t bytes = packed_bytes(count); + const uint8_t* chunk_input = input + input_bit_offset / 8; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svint16_t unpacked = + sve2_unpack_epi16_9to16(svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); + + sve2_store_dequantized_i16_f32(unpacked, scale_v, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcntw()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const svfloat32_t scale_v = svdup_n_f32(scale); + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint8_t* chunk_input = input + input_bit_offset / 8; + const uint32_t bit_offset = input_bit_offset % 8; + const uint32_t bytes = (bit_offset + count * BIT_WIDTH + 7u) / 8u; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + svint32_t unpacked; + if (bit_offset == 0) { + unpacked = sve2_unpack_epi32_17to24(source); + } else { + unpacked = sve2_unpack_epi32_17to24(source); + } + + sve2_store_dequantized_i32_f32(unpacked, scale_v, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__, double_t, double_t* __restrict__) { return -1; } template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_block(const uint8_t*, double_t, double_t*) { - static_assert(internal::sve2_decompression_unimplemented_v, "ARM64 SVE2 decompression is not implemented yet"); + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcnth()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const table_unpacking table; + const svuint8_t permute = table.permute(); + const svuint16_t shift = table.shift(); + svuint8_t spill_permute = svdup_n_u8(0); + svuint16_t spill_shift = svdup_n_u16(0); + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + spill_permute = table.spill_permute(); + spill_shift = table.spill_shift(); + } + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint32_t bytes = packed_bytes(count); + const uint8_t* chunk_input = input + input_bit_offset / 8; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svint16_t unpacked = + sve2_unpack_epi16_9to16(svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); + + sve2_store_dequantized_i16_f64(unpacked, scale, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__, double_t, double_t* __restrict__) { return -1; } +} // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_blocks(const uint8_t*, float_t, float_t*, uint32_t) { - static_assert(internal::sve2_decompression_unimplemented_v, "ARM64 SVE2 decompression is not implemented yet"); - return -1; +int sve2_decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::sve2_decompress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::sve2_decompress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::sve2_decompress_block_17to24(input, scale, output); + } } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_blocks(const uint8_t*, double_t, double_t*, uint32_t) { - static_assert(internal::sve2_decompression_unimplemented_v, "ARM64 SVE2 decompression is not implemented yet"); - return -1; +int sve2_decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::sve2_decompress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::sve2_decompress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::sve2_decompress_block_17to24(input, scale, output); + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { + if constexpr (BIT_WIDTH > 8) { + return -1; + } else { + const uint8_t* block_input = input; + float_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + sve2_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { + if constexpr (BIT_WIDTH > 8) { + return -1; + } else { + const uint8_t* block_input = input; + double_t* block_output = output; + + for (uint32_t block = 0; block < blocks; ++block) { + sve2_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; + } } #ifdef __cplusplus @@ -45,15 +354,18 @@ extern "C" { #endif int sve2_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); + int sve2_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); + int sve2_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); + int sve2_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, uint32_t blocks); #ifdef __cplusplus } #endif -} // namespace pernix +} // namespace pernix::arm64::sve2 #endif // PERNIX_ARM64_SVE2_DECOMPRESSION_H diff --git a/include/pernix/arm64/sve2/tables.h b/include/pernix/arm64/sve2/tables.h new file mode 100644 index 0000000..813e042 --- /dev/null +++ b/include/pernix/arm64/sve2/tables.h @@ -0,0 +1,143 @@ +#ifndef PERNIX_ARM64_SVE2_TABLES_H +#define PERNIX_ARM64_SVE2_TABLES_H + +#include + +#include +#include + +namespace pernix::arm64::sve2::internal { +template +struct table_unpacking { + static constexpr uint8_t bit_width = BIT_WIDTH; + + static svbool_t pg_b8() { return svptrue_b8(); } + + static svbool_t pg_b16() { return svptrue_b16(); } + + static svbool_t pg_b32() { return svptrue_b32(); } +}; + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +struct table_unpacking { + static constexpr uint8_t bit_width = BIT_WIDTH; + + static svuint8_t permute() { + std::vector table(svcntb()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + table[lane] = static_cast((lane * BIT_WIDTH) / 8u); + } + + return svld1_u8(svptrue_b8(), table.data()); + } + + static svuint8_t spill_permute() { + std::vector table(svcntb()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + table[lane] = static_cast((lane * BIT_WIDTH) / 8u + 1u); + } + + return svld1_u8(svptrue_b8(), table.data()); + } + + static svuint8_t shift() { + std::vector table(svcntb()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + table[lane] = static_cast((lane * BIT_WIDTH) % 8u); + } + + return svld1_u8(svptrue_b8(), table.data()); + } + + static svuint8_t spill_shift() { + std::vector table(svcntb()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + table[lane] = static_cast(8u - ((lane * BIT_WIDTH) % 8u)); + } + + return svld1_u8(svptrue_b8(), table.data()); + } +}; + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +struct table_unpacking { + static constexpr uint8_t bit_width = BIT_WIDTH; + + static svuint8_t permute() { + std::vector table(svcntb()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + const uint32_t element = lane / 2u; + const uint32_t byte = lane % 2u; + const uint32_t first = (element * BIT_WIDTH) / 8u; + + table[lane] = static_cast(first + byte); + } + + return svld1_u8(svptrue_b8(), table.data()); + } + + static svuint8_t spill_permute() { + std::vector table(svcntb()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + const uint32_t element = lane / 2u; + const uint32_t byte = lane % 2u; + const uint32_t first = (element * BIT_WIDTH) / 8u; + + table[lane] = static_cast(first + 2u + byte); + } + + return svld1_u8(svptrue_b8(), table.data()); + } + + static svuint16_t shift() { + std::vector table(svcnth()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + table[lane] = static_cast((lane * BIT_WIDTH) % 8u); + } + + return svld1_u16(svptrue_b16(), table.data()); + } + + static svuint16_t spill_shift() { + std::vector table(svcnth()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + const uint32_t bit_offset = (lane * BIT_WIDTH) % 8u; + table[lane] = bit_offset + BIT_WIDTH > 16u ? static_cast(16u - bit_offset) : uint16_t{16}; + } + + return svld1_u16(svptrue_b16(), table.data()); + } +}; + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && START_BIT_OFFSET < 8) +struct table_unpacking { + static constexpr uint8_t bit_width = BIT_WIDTH; + + static svuint8_t permute() { + std::vector table(svcntb()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + const uint32_t element = lane / 4u; + const uint32_t byte = lane % 4u; + const uint32_t first = (START_BIT_OFFSET + element * BIT_WIDTH) / 8u; + + table[lane] = static_cast(first + byte); + } + + return svld1_u8(svptrue_b8(), table.data()); + } + + static svuint32_t shift() { + std::vector table(svcntw()); + for (uint32_t lane = 0; lane < table.size(); ++lane) { + table[lane] = (START_BIT_OFFSET + lane * BIT_WIDTH) % 8u; + } + + return svld1_u32(svptrue_b32(), table.data()); + } +}; +} // namespace pernix::arm64::sve2::internal + +#endif // PERNIX_ARM64_SVE2_TABLES_H diff --git a/include/pernix/arm64/sve2/unpacking.h b/include/pernix/arm64/sve2/unpacking.h index d654b5e..326901f 100644 --- a/include/pernix/arm64/sve2/unpacking.h +++ b/include/pernix/arm64/sve2/unpacking.h @@ -3,9 +3,90 @@ #include +#include "tables.h" + namespace pernix::arm64::sve2::internal { -template -inline constexpr bool unpacking_unimplemented_v = false; -} // namespace pernix::arm64::sve2::internal +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline svint8_t sve2_unpack_epi8_1to8(const svuint8_t input, const svuint8_t permute, const svuint8_t shift, + const svuint8_t spill_permute, const svuint8_t spill_shift) { + if constexpr (BIT_WIDTH == 8) { + return svreinterpret_s8(input); + } else { + const svbool_t pg = svptrue_b8(); + + const svuint8_t permuted = svtbl_u8(input, permute); + svuint8_t unpacked = svlsr_u8_x(pg, permuted, shift); + + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + const svuint8_t spill_permuted_values = svtbl_u8(input, spill_permute); + const svuint8_t spill_shifted = svlsl_u8_x(pg, spill_permuted_values, spill_shift); + unpacked = svorr_u8_x(pg, unpacked, spill_shifted); + } + + if constexpr (BIT_WIDTH == 1) { + unpacked = svand_n_u8_x(pg, unpacked, 1); + return svreinterpret_s8(unpacked); + } else { + constexpr int sign_shift = 8 - BIT_WIDTH; + + unpacked = svlsl_n_u8_x(pg, unpacked, sign_shift); + + if constexpr (SIGN_VALUES) { + return svasr_n_s8_x(pg, svreinterpret_s8_u8(unpacked), sign_shift); + } else { + return svreinterpret_s8_u8(svlsr_n_u8_x(pg, unpacked, sign_shift)); + } + } + } +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline svint16_t sve2_unpack_epi16_9to16(const svuint16_t input, const svuint8_t permute, const svuint16_t shift, + const svuint8_t spill_permute, const svuint16_t spill_shift) { + if constexpr (BIT_WIDTH == 16) { + return svreinterpret_s16(input); + } else { + const svbool_t pg = svptrue_b16(); + + const svuint8_t permuted = svtbl_u8(svreinterpret_u8_u16(input), permute); + svuint16_t shifted = svlsr_u16_x(pg, svreinterpret_u16_u8(permuted), shift); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const svuint8_t spill_permuted_values = svtbl_u8(svreinterpret_u8_u16(input), spill_permute); + const svuint16_t spill_shifted = svlsl_u16_x(pg, svreinterpret_u16_u8(spill_permuted_values), spill_shift); + shifted = svorr_u16_x(pg, shifted, spill_shifted); + } + + constexpr int sign_shift = 16 - BIT_WIDTH; + shifted = svlsl_n_u16_x(pg, shifted, sign_shift); + + if constexpr (SIGN_VALUES) { + return svasr_n_s16_x(pg, svreinterpret_s16_u16(shifted), sign_shift); + } else { + return svreinterpret_s16_u16(svlsr_n_u16_x(pg, shifted, sign_shift)); + } + } +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline svint32_t sve2_unpack_epi32_17to24(const svuint8_t input) { + using table = table_unpacking; + + const svbool_t pg = svptrue_b32(); + const svuint8_t permuted = svtbl_u8(input, table::permute()); + const svuint32_t unpacked = svlsr_u32_x(pg, svreinterpret_u32_u8(permuted), table::shift()); + + if constexpr (SIGN_VALUES) { + constexpr int sign_shift = 32 - BIT_WIDTH; + return svasr_n_s32_x(pg, svreinterpret_s32_u32(svlsl_n_u32_x(pg, unpacked, sign_shift)), sign_shift); + } else { + constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1u; + return svreinterpret_s32_u32(svand_n_u32_x(pg, unpacked, mask)); + } +} +} // namespace pernix::arm64::sve2::internal #endif // PERNIX_ARM64_SVE2_UNPACKING_H diff --git a/include/pernix/simd_compat.h b/include/pernix/simd_compat.h index 07d5110..509eb13 100644 --- a/include/pernix/simd_compat.h +++ b/include/pernix/simd_compat.h @@ -37,8 +37,13 @@ #elif defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) #include #elif defined(__aarch64__) || defined(__arm64ec__) +#ifdef __ARM_FEATURE_SVE +#include +#endif +#ifdef __ARM_NEON #include #endif +#endif #ifndef __always_inline #if defined(__GNUC__) || defined(__clang__) diff --git a/src/arm64/neon/decompression.cpp b/src/arm64/neon/decompression.cpp index 3cb7fc7..a89f763 100644 --- a/src/arm64/neon/decompression.cpp +++ b/src/arm64/neon/decompression.cpp @@ -2,20 +2,142 @@ namespace pernix { extern "C" { -int neon_decompress_block(uint8_t, const uint8_t*, float_t, float_t*) { - return -1; +#define PERNIX_NEON_DECOMPRESS_BLOCK_CASE(N) \ + case N: \ + return arm64::neon::neon_decompress_block(input, scale, output); + +#define PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(N) \ + case N: \ + return arm64::neon::neon_decompress_blocks(input, scale, output, blocks); + +int neon_decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + switch (bit_width) { + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(1) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(2) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(3) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(4) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(5) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(6) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(7) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(8) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(9) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(10) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(11) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(12) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(13) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(14) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(15) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(16) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(17) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(18) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(19) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(20) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(21) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(22) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(23) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(24) + default: + return -1; + } } -int neon_decompress_block_f64(uint8_t, const uint8_t*, double_t, double_t*) { - return -1; +int neon_decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + switch (bit_width) { + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(1) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(2) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(3) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(4) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(5) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(6) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(7) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(8) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(9) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(10) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(11) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(12) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(13) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(14) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(15) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(16) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(17) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(18) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(19) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(20) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(21) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(22) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(23) + PERNIX_NEON_DECOMPRESS_BLOCK_CASE(24) + default: + return -1; + } } -int neon_decompress_blocks(uint8_t, const uint8_t*, float_t, float_t*, uint32_t) { - return -1; +int neon_decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, + const uint32_t blocks) { + switch (bit_width) { + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(1) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(2) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(3) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(4) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(5) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(6) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(7) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(8) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(9) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(10) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(11) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(12) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(13) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(14) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(15) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(16) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(17) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(18) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(19) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(20) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(21) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(22) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(23) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(24) + default: + return -1; + } } -int neon_decompress_blocks_f64(uint8_t, const uint8_t*, double_t, double_t*, uint32_t) { - return -1; +int neon_decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output, const uint32_t blocks) { + switch (bit_width) { + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(1) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(2) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(3) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(4) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(5) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(6) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(7) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(8) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(9) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(10) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(11) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(12) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(13) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(14) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(15) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(16) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(17) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(18) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(19) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(20) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(21) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(22) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(23) + PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(24) + default: + return -1; + } } + +#undef PERNIX_NEON_DECOMPRESS_BLOCK_CASE +#undef PERNIX_NEON_DECOMPRESS_BLOCKS_CASE } -} // namespace pernix +} // namespace pernix diff --git a/src/arm64/sve2/decompression.cpp b/src/arm64/sve2/decompression.cpp index 8d170f6..66bbd97 100644 --- a/src/arm64/sve2/decompression.cpp +++ b/src/arm64/sve2/decompression.cpp @@ -2,20 +2,78 @@ namespace pernix { extern "C" { -int sve2_decompress_block(uint8_t, const uint8_t*, float_t, float_t*) { - return -1; +#define PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(N) \ + case N: \ + return arm64::sve2::sve2_decompress_block(input, scale, output); + +#define PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(N) \ + case N: \ + return arm64::sve2::sve2_decompress_blocks(input, scale, output, blocks); + +int sve2_decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + switch (bit_width) { + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(1) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(2) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(3) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(4) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(5) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(6) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(7) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(8) + default: + return -1; + } } -int sve2_decompress_block_f64(uint8_t, const uint8_t*, double_t, double_t*) { - return -1; +int sve2_decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { + switch (bit_width) { + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(1) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(2) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(3) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(4) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(5) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(6) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(7) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(8) + default: + return -1; + } } -int sve2_decompress_blocks(uint8_t, const uint8_t*, float_t, float_t*, uint32_t) { - return -1; +int sve2_decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, + const uint32_t blocks) { + switch (bit_width) { + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(1) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(2) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(3) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(4) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(5) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(6) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(7) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(8) + default: + return -1; + } } -int sve2_decompress_blocks_f64(uint8_t, const uint8_t*, double_t, double_t*, uint32_t) { - return -1; +int sve2_decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output, const uint32_t blocks) { + switch (bit_width) { + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(1) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(2) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(3) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(4) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(5) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(6) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(7) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(8) + default: + return -1; + } } + +#undef PERNIX_SVE2_DECOMPRESS_BLOCK_CASE +#undef PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE } -} // namespace pernix +} // namespace pernix diff --git a/tests/arm64/sve2/.gitkeep b/tests/arm64/sve2/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tests/arm64/sve2/decompression_tests.cpp b/tests/arm64/sve2/decompression_tests.cpp new file mode 100644 index 0000000..82cb11f --- /dev/null +++ b/tests/arm64/sve2/decompression_tests.cpp @@ -0,0 +1,38 @@ +#include +#include + +#ifdef PERNIX_BACKEND_ARM64_SVE2 + +using namespace pernix::arm64::sve2; + +TYPED_TEST(DecompressionTest, SVE2DecompressBlock) { + std::vector > decompressedData(this->testSet.numberOfBlocks); + + for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { + decompressedData[block].resize(this->testSet.elementsPerBlock); + + sve2_decompress_block( + this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); + } + + for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { + expectDecompressedBlockNearSource(*this, decompressedData[block], block); + } +} + +TYPED_TEST(DecompressionTest64, SVE2DecompressBlock) { + std::vector > decompressedData(this->testSet.numberOfBlocks); + + for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { + decompressedData[block].resize(this->testSet.elementsPerBlock); + + sve2_decompress_block( + this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); + } + + for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { + expectDecompressedBlockNearSource(*this, decompressedData[block], block); + } +} + +#endif \ No newline at end of file From 0f0c1782b66c6687948714a6a53f6183b633d972 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 28 May 2026 21:32:10 +0200 Subject: [PATCH 11/43] WIP: Implement SVE2 decompression functions --- include/pernix/arm64/sve2/decompression.h | 122 +++++++++++++++++----- 1 file changed, 94 insertions(+), 28 deletions(-) diff --git a/include/pernix/arm64/sve2/decompression.h b/include/pernix/arm64/sve2/decompression.h index 198a8fc..2128ff2 100644 --- a/include/pernix/arm64/sve2/decompression.h +++ b/include/pernix/arm64/sve2/decompression.h @@ -122,6 +122,22 @@ __always_inline void sve2_store_dequantized_i32_f32(svint32_t values, const svfl svst1_f32(pg, output, dequantized); } +template +__always_inline void sve2_store_dequantized_i32_f64(svint32_t values, const double_t scale, double_t* __restrict__ output, + const uint32_t count) { + std::vector temp(svcntw()); + + svst1_s32(svptrue_b32(), temp.data(), values); + + for (uint32_t i = 0; i < count; ++i) { + if constexpr (SIGN_VALUES) { + output[i] = static_cast(temp[i]) * scale; + } else { + output[i] = static_cast(static_cast(temp[i])) * scale; + } + } +} + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { @@ -238,8 +254,39 @@ __always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ inp template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__, double_t, double_t* __restrict__) { - return -1; +__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcntb()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const table_unpacking table; + const svuint8_t permute = table.permute(); + const svuint8_t shift = table.shift(); + svuint8_t spill_permute = svdup_n_u8(0); + svuint8_t spill_shift = svdup_n_u8(0); + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + spill_permute = table.spill_permute(); + spill_shift = table.spill_shift(); + } + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint32_t bytes = packed_bytes(count); + const uint8_t* chunk_input = input + input_bit_offset / 8; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svint8_t unpacked = sve2_unpack_epi8_1to8(source, permute, shift, spill_permute, spill_shift); + + sve2_store_dequantized_i8_f64(unpacked, scale, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; } template @@ -282,8 +329,35 @@ __always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ inpu template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__, double_t, double_t* __restrict__) { - return -1; +__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcntw()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint8_t* chunk_input = input + input_bit_offset / 8; + const uint32_t bit_offset = input_bit_offset % 8; + const uint32_t bytes = (bit_offset + count * BIT_WIDTH + 7u) / 8u; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + svint32_t unpacked; + if (bit_offset == 0) { + unpacked = sve2_unpack_epi32_17to24(source); + } else { + unpacked = sve2_unpack_epi32_17to24(source); + } + + sve2_store_dequantized_i32_f64(unpacked, scale, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; } } // namespace internal @@ -314,39 +388,31 @@ int sve2_decompress_block(const uint8_t* __restrict__ input, const double_t scal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - if constexpr (BIT_WIDTH > 8) { - return -1; - } else { - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - sve2_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + const uint8_t* block_input = input; + float_t* block_output = output; - return 0; + for (uint32_t block = 0; block < blocks; ++block) { + sve2_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } + + return 0; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - if constexpr (BIT_WIDTH > 8) { - return -1; - } else { - const uint8_t* block_input = input; - double_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - sve2_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + const uint8_t* block_input = input; + double_t* block_output = output; - return 0; + for (uint32_t block = 0; block < blocks; ++block) { + sve2_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } + + return 0; } #ifdef __cplusplus From 95c6475d8092f96dd3417c376bcda8e03ba9e94e Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 28 May 2026 21:59:45 +0200 Subject: [PATCH 12/43] WIP: Extend SVE2 decompression functions --- include/pernix/arm64/sve2/tables.h | 115 +++++++++++------------------ src/arm64/sve2/decompression.cpp | 64 ++++++++++++++++ 2 files changed, 109 insertions(+), 70 deletions(-) diff --git a/include/pernix/arm64/sve2/tables.h b/include/pernix/arm64/sve2/tables.h index 813e042..897fa9b 100644 --- a/include/pernix/arm64/sve2/tables.h +++ b/include/pernix/arm64/sve2/tables.h @@ -4,7 +4,6 @@ #include #include -#include namespace pernix::arm64::sve2::internal { template @@ -24,39 +23,23 @@ struct table_unpacking { static constexpr uint8_t bit_width = BIT_WIDTH; static svuint8_t permute() { - std::vector table(svcntb()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - table[lane] = static_cast((lane * BIT_WIDTH) / 8u); - } - - return svld1_u8(svptrue_b8(), table.data()); + const svbool_t pg = svptrue_b8(); + return svlsr_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 3); } static svuint8_t spill_permute() { - std::vector table(svcntb()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - table[lane] = static_cast((lane * BIT_WIDTH) / 8u + 1u); - } - - return svld1_u8(svptrue_b8(), table.data()); + const svbool_t pg = svptrue_b8(); + return svadd_n_u8_x(pg, permute(), 1); } static svuint8_t shift() { - std::vector table(svcntb()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - table[lane] = static_cast((lane * BIT_WIDTH) % 8u); - } - - return svld1_u8(svptrue_b8(), table.data()); + const svbool_t pg = svptrue_b8(); + return svand_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 7); } static svuint8_t spill_shift() { - std::vector table(svcntb()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - table[lane] = static_cast(8u - ((lane * BIT_WIDTH) % 8u)); - } - - return svld1_u8(svptrue_b8(), table.data()); + const svbool_t pg = svptrue_b8(); + return svsub_u8_x(pg, svdup_n_u8(8), shift()); } }; @@ -66,48 +49,39 @@ struct table_unpacking { static constexpr uint8_t bit_width = BIT_WIDTH; static svuint8_t permute() { - std::vector table(svcntb()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - const uint32_t element = lane / 2u; - const uint32_t byte = lane % 2u; - const uint32_t first = (element * BIT_WIDTH) / 8u; - - table[lane] = static_cast(first + byte); + const svbool_t pg = svptrue_b8(); + const svuint8_t lane = svindex_u8(0, 1); + const svuint8_t elem = svlsr_n_u8_x(pg, lane, 1); + const svuint8_t byte = svand_n_u8_x(pg, lane, 1); + + svuint8_t first; + if constexpr (BIT_WIDTH == 16) { + first = svlsl_n_u8_x(pg, elem, 1); + } else { + constexpr uint8_t extra_bits = BIT_WIDTH - 8u; + const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); + const svuint8_t low = svlsr_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), 3); + first = svadd_u8_x(pg, elem, svadd_u8_x(pg, high, low)); } - return svld1_u8(svptrue_b8(), table.data()); + return svadd_u8_x(pg, first, byte); } static svuint8_t spill_permute() { - std::vector table(svcntb()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - const uint32_t element = lane / 2u; - const uint32_t byte = lane % 2u; - const uint32_t first = (element * BIT_WIDTH) / 8u; - - table[lane] = static_cast(first + 2u + byte); - } - - return svld1_u8(svptrue_b8(), table.data()); + const svbool_t pg = svptrue_b8(); + return svadd_n_u8_x(pg, permute(), 2); } static svuint16_t shift() { - std::vector table(svcnth()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - table[lane] = static_cast((lane * BIT_WIDTH) % 8u); - } - - return svld1_u16(svptrue_b16(), table.data()); + const svbool_t pg = svptrue_b16(); + return svand_n_u16_x(pg, svmul_n_u16_x(pg, svindex_u16(0, 1), BIT_WIDTH), 7); } static svuint16_t spill_shift() { - std::vector table(svcnth()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - const uint32_t bit_offset = (lane * BIT_WIDTH) % 8u; - table[lane] = bit_offset + BIT_WIDTH > 16u ? static_cast(16u - bit_offset) : uint16_t{16}; - } - - return svld1_u16(svptrue_b16(), table.data()); + const svbool_t pg = svptrue_b16(); + const svuint16_t bit_shift = shift(); + const svuint16_t spill = svsub_u16_x(pg, svdup_n_u16(16), bit_shift); + return svsel_u16(svcmpgt_n_u16(pg, bit_shift, 16u - BIT_WIDTH), spill, svdup_n_u16(16)); } }; @@ -117,25 +91,26 @@ struct table_unpacking { static constexpr uint8_t bit_width = BIT_WIDTH; static svuint8_t permute() { - std::vector table(svcntb()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - const uint32_t element = lane / 4u; - const uint32_t byte = lane % 4u; - const uint32_t first = (START_BIT_OFFSET + element * BIT_WIDTH) / 8u; - - table[lane] = static_cast(first + byte); + const svbool_t pg = svptrue_b8(); + const svuint8_t lane = svindex_u8(0, 1); + const svuint8_t elem = svlsr_n_u8_x(pg, lane, 2); + const svuint8_t byte = svand_n_u8_x(pg, lane, 3); + + svuint8_t first = svmul_n_u8_x(pg, elem, BIT_WIDTH / 8u); + if constexpr (BIT_WIDTH % 8u != 0) { + constexpr uint8_t extra_bits = BIT_WIDTH % 8u; + const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); + const svuint8_t low_bits = + svadd_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), START_BIT_OFFSET); + first = svadd_u8_x(pg, first, svadd_u8_x(pg, high, svlsr_n_u8_x(pg, low_bits, 3))); } - return svld1_u8(svptrue_b8(), table.data()); + return svadd_u8_x(pg, first, byte); } static svuint32_t shift() { - std::vector table(svcntw()); - for (uint32_t lane = 0; lane < table.size(); ++lane) { - table[lane] = (START_BIT_OFFSET + lane * BIT_WIDTH) % 8u; - } - - return svld1_u32(svptrue_b32(), table.data()); + const svbool_t pg = svptrue_b32(); + return svand_n_u32_x(pg, svadd_n_u32_x(pg, svmul_n_u32_x(pg, svindex_u32(0, 1), BIT_WIDTH), START_BIT_OFFSET), 7); } }; } // namespace pernix::arm64::sve2::internal diff --git a/src/arm64/sve2/decompression.cpp b/src/arm64/sve2/decompression.cpp index 66bbd97..8429e97 100644 --- a/src/arm64/sve2/decompression.cpp +++ b/src/arm64/sve2/decompression.cpp @@ -20,6 +20,22 @@ int sve2_decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ i PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(6) PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(7) PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(8) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(9) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(10) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(11) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(12) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(13) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(14) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(15) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(16) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(17) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(18) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(19) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(20) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(21) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(22) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(23) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(24) default: return -1; } @@ -36,6 +52,22 @@ int sve2_decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(6) PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(7) PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(8) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(9) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(10) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(11) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(12) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(13) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(14) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(15) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(16) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(17) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(18) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(19) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(20) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(21) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(22) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(23) + PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(24) default: return -1; } @@ -52,6 +84,22 @@ int sve2_decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(6) PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(7) PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(8) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(9) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(10) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(11) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(12) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(13) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(14) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(15) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(16) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(17) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(18) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(19) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(20) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(21) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(22) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(23) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(24) default: return -1; } @@ -68,6 +116,22 @@ int sve2_decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restric PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(6) PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(7) PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(8) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(9) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(10) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(11) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(12) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(13) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(14) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(15) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(16) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(17) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(18) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(19) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(20) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(21) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(22) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(23) + PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(24) default: return -1; } From 5523a3d4a6ad8bef90ecb04082294a76bda8c76e Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Sat, 13 Jun 2026 14:20:50 +0200 Subject: [PATCH 13/43] feat: add runtime backend dispatch with configurable block sizes --- CMakeLists.txt | 152 ++-- cmake/pernixConfig.cmake.in | 18 + include/pernix/arm64/neon/common.h | 200 ----- include/pernix/arm64/neon/unpacking.h | 100 --- include/pernix/arm64/sve/compression.h | 141 ---- include/pernix/arm64/sve/decompression.h | 141 ---- include/pernix/arm64/sve/packing.h | 9 - include/pernix/arm64/sve/unpacking.h | 9 - include/pernix/arm64/sve2/tables.h | 118 --- include/pernix/arm64/sve2/unpacking.h | 92 --- include/pernix/compat.h | 24 + include/pernix/detection.h | 87 --- include/pernix/fallback/decompression.h | 286 -------- include/pernix/pernix.h | 602 +-------------- include/pernix/pernix.hpp | 153 ++++ include/pernix/x86/avx512vbmi/compat.h | 387 ---------- include/pernix/x86/avx512vbmi/packing.h | 327 --------- include/pernix/x86/avx512vbmi/unpacking.h | 500 ------------- src/CMakeLists.txt | 210 +++++- src/arm64/neon/compression.cpp | 29 +- src/arm64/neon/decompression.cpp | 310 ++++---- src/arm64/sve/compression.cpp | 21 - src/arm64/sve/decompression.cpp | 21 - src/arm64/sve2/compression.cpp | 29 +- src/arm64/sve2/decompression.cpp | 310 ++++---- src/dispatch/cpu_features_arm.cpp | 23 + src/dispatch/cpu_features_x86.cpp | 95 +++ src/dispatch/select.cpp | 684 ++++++++++++++++++ src/fallback/compression.cpp | 149 ---- src/fallback/decompression.cpp | 150 ---- src/fallback/fallback_compression.cpp | 194 +++++ src/fallback/fallback_decompression.cpp | 201 +++++ src/internal/pernix/arm64/neon/common.h | 223 ++++++ .../internal}/pernix/arm64/neon/compression.h | 40 +- .../pernix/arm64/neon/decompression.h | 147 ++-- .../internal}/pernix/arm64/neon/packing.h | 0 .../internal}/pernix/arm64/neon/tables.h | 0 src/internal/pernix/arm64/neon/unpacking.h | 101 +++ .../internal}/pernix/arm64/sve2/compression.h | 35 +- .../pernix/arm64/sve2/decompression.h | 161 +++-- .../internal}/pernix/arm64/sve2/packing.h | 4 +- src/internal/pernix/arm64/sve2/tables.h | 119 +++ src/internal/pernix/arm64/sve2/unpacking.h | 94 +++ src/internal/pernix/dispatch/cpu_features.h | 26 + src/internal/pernix/dispatch/kernel.h | 27 + src/internal/pernix/dispatch/select.h | 159 ++++ .../pernix/fallback/avx2_compression.h | 89 +-- .../pernix/fallback/avx2_decompression.h | 248 +++++++ .../internal}/pernix/simd_compat.h | 11 +- .../pernix/x86/avx2/avx2_compression.h | 368 +++++----- .../pernix/x86/avx2/avx2_decompression.h | 98 +-- .../internal/pernix/x86/avx2/avx2_tables.h | 341 ++++----- .../x86/avx512vbmi/avx512vbmi_compression.h | 113 +-- .../x86/avx512vbmi/avx512vbmi_decompression.h | 112 +-- src/internal/pernix/x86/avx512vbmi/compat.h | 387 ++++++++++ src/internal/pernix/x86/avx512vbmi/packing.h | 331 +++++++++ .../internal}/pernix/x86/avx512vbmi/tables.h | 533 +++++++------- .../pernix/x86/avx512vbmi/unpacking.h | 500 +++++++++++++ .../pernix/x86/bmi2/bmi2_compression.h | 240 +++--- .../pernix/x86/bmi2/bmi2_decompression.h | 96 +-- {include => src/internal}/pernix/x86/utils.h | 0 src/pernix.cpp | 293 +++----- src/x86/avx2/avx2_compression.cpp | 193 +++++ src/x86/avx2/avx2_decompression.cpp | 201 +++++ src/x86/avx2/compression.cpp | 154 ---- src/x86/avx2/decompression.cpp | 153 ---- src/x86/avx512vbmi/avx512vbmi_compression.cpp | 193 +++++ .../avx512vbmi/avx512vbmi_decompression.cpp | 201 +++++ src/x86/avx512vbmi/compression.cpp | 153 ---- src/x86/avx512vbmi/decompression.cpp | 153 ---- src/x86/bmi2/bmi2_compression.cpp | 193 +++++ src/x86/bmi2/bmi2_decompression.cpp | 201 +++++ src/x86/bmi2/compression.cpp | 153 ---- src/x86/bmi2/decompression.cpp | 153 ---- tests/CMakeLists.txt | 60 +- tests/arm64/neon/decompression_tests.cpp | 38 - tests/arm64/sve/.gitkeep | 0 tests/arm64/sve2/decompression_tests.cpp | 38 - tests/fallback/compression_tests.cpp | 34 - tests/fallback/decompression_tests.cpp | 32 - tests/fallback/edge_tests.cpp | 44 -- tests/fallback_tests.cpp | 316 ++++++++ tests/include/testset.h | 34 +- tests/simd_tests.cpp | 188 +++++ tests/x86/avx2/compression_tests.cpp | 46 -- tests/x86/avx2/decompression_tests.cpp | 36 - tests/x86/avx512vbmi/compression_tests.cpp | 46 -- tests/x86/avx512vbmi/decompression_tests.cpp | 36 - tests/x86/bmi2/compression_tests.cpp | 46 -- tests/x86/bmi2/decompression_tests.cpp | 36 - 90 files changed, 7158 insertions(+), 6641 deletions(-) create mode 100644 cmake/pernixConfig.cmake.in delete mode 100644 include/pernix/arm64/neon/common.h delete mode 100644 include/pernix/arm64/neon/unpacking.h delete mode 100644 include/pernix/arm64/sve/compression.h delete mode 100644 include/pernix/arm64/sve/decompression.h delete mode 100644 include/pernix/arm64/sve/packing.h delete mode 100644 include/pernix/arm64/sve/unpacking.h delete mode 100644 include/pernix/arm64/sve2/tables.h delete mode 100644 include/pernix/arm64/sve2/unpacking.h create mode 100644 include/pernix/compat.h delete mode 100644 include/pernix/detection.h delete mode 100644 include/pernix/fallback/decompression.h create mode 100644 include/pernix/pernix.hpp delete mode 100644 include/pernix/x86/avx512vbmi/compat.h delete mode 100644 include/pernix/x86/avx512vbmi/packing.h delete mode 100644 include/pernix/x86/avx512vbmi/unpacking.h delete mode 100644 src/arm64/sve/compression.cpp delete mode 100644 src/arm64/sve/decompression.cpp create mode 100644 src/dispatch/cpu_features_arm.cpp create mode 100644 src/dispatch/cpu_features_x86.cpp create mode 100644 src/dispatch/select.cpp delete mode 100644 src/fallback/compression.cpp delete mode 100644 src/fallback/decompression.cpp create mode 100644 src/fallback/fallback_compression.cpp create mode 100644 src/fallback/fallback_decompression.cpp create mode 100644 src/internal/pernix/arm64/neon/common.h rename {include => src/internal}/pernix/arm64/neon/compression.h (79%) rename {include => src/internal}/pernix/arm64/neon/decompression.h (77%) rename {include => src/internal}/pernix/arm64/neon/packing.h (100%) rename {include => src/internal}/pernix/arm64/neon/tables.h (100%) create mode 100644 src/internal/pernix/arm64/neon/unpacking.h rename {include => src/internal}/pernix/arm64/sve2/compression.h (52%) rename {include => src/internal}/pernix/arm64/sve2/decompression.h (76%) rename {include => src/internal}/pernix/arm64/sve2/packing.h (74%) create mode 100644 src/internal/pernix/arm64/sve2/tables.h create mode 100644 src/internal/pernix/arm64/sve2/unpacking.h create mode 100644 src/internal/pernix/dispatch/cpu_features.h create mode 100644 src/internal/pernix/dispatch/kernel.h create mode 100644 src/internal/pernix/dispatch/select.h rename include/pernix/fallback/compression.h => src/internal/pernix/fallback/avx2_compression.h (73%) create mode 100644 src/internal/pernix/fallback/avx2_decompression.h rename {include => src/internal}/pernix/simd_compat.h (82%) rename include/pernix/x86/avx2/compression.h => src/internal/pernix/x86/avx2/avx2_compression.h (67%) rename include/pernix/x86/avx2/decompression.h => src/internal/pernix/x86/avx2/avx2_decompression.h (79%) rename include/pernix/x86/avx2/tables.h => src/internal/pernix/x86/avx2/avx2_tables.h (68%) rename include/pernix/x86/avx512vbmi/compression.h => src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h (88%) rename include/pernix/x86/avx512vbmi/decompression.h => src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h (87%) create mode 100644 src/internal/pernix/x86/avx512vbmi/compat.h create mode 100644 src/internal/pernix/x86/avx512vbmi/packing.h rename {include => src/internal}/pernix/x86/avx512vbmi/tables.h (51%) create mode 100644 src/internal/pernix/x86/avx512vbmi/unpacking.h rename include/pernix/x86/bmi2/compression.h => src/internal/pernix/x86/bmi2/bmi2_compression.h (56%) rename include/pernix/x86/bmi2/decompression.h => src/internal/pernix/x86/bmi2/bmi2_decompression.h (80%) rename {include => src/internal}/pernix/x86/utils.h (100%) create mode 100644 src/x86/avx2/avx2_compression.cpp create mode 100644 src/x86/avx2/avx2_decompression.cpp delete mode 100644 src/x86/avx2/compression.cpp delete mode 100644 src/x86/avx2/decompression.cpp create mode 100644 src/x86/avx512vbmi/avx512vbmi_compression.cpp create mode 100644 src/x86/avx512vbmi/avx512vbmi_decompression.cpp delete mode 100644 src/x86/avx512vbmi/compression.cpp delete mode 100644 src/x86/avx512vbmi/decompression.cpp create mode 100644 src/x86/bmi2/bmi2_compression.cpp create mode 100644 src/x86/bmi2/bmi2_decompression.cpp delete mode 100644 src/x86/bmi2/compression.cpp delete mode 100644 src/x86/bmi2/decompression.cpp delete mode 100644 tests/arm64/neon/decompression_tests.cpp delete mode 100644 tests/arm64/sve/.gitkeep delete mode 100644 tests/arm64/sve2/decompression_tests.cpp delete mode 100644 tests/fallback/compression_tests.cpp delete mode 100644 tests/fallback/decompression_tests.cpp delete mode 100644 tests/fallback/edge_tests.cpp create mode 100644 tests/fallback_tests.cpp create mode 100644 tests/simd_tests.cpp delete mode 100644 tests/x86/avx2/compression_tests.cpp delete mode 100644 tests/x86/avx2/decompression_tests.cpp delete mode 100644 tests/x86/avx512vbmi/compression_tests.cpp delete mode 100644 tests/x86/avx512vbmi/decompression_tests.cpp delete mode 100644 tests/x86/bmi2/compression_tests.cpp delete mode 100644 tests/x86/bmi2/decompression_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1cbead6..c6af942 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.30) +cmake_minimum_required(VERSION 3.24) project(pernix VERSION 0.1.0 LANGUAGES CXX C) option(PERNIX_ENABLE_TESTS "Enable tests for pernix" on) @@ -8,44 +8,19 @@ option(PERNIX_ENABLE_INSTALL "Enable installation of pernix" on) option(PERNIX_ENABLE_DOXYGEN "Build documentation with Doxygen." off) option(PERNIX_INSTALL_DOCS "Install documentation with pernix" off) -option(PERNIX_DISABLE_BMI2 "Disable BMI2 optimizations" off) -option(PERNIX_DISABLE_AVX2 "Disable AVX2 optimizations" off) -option(PERNIX_DISABLE_AVX512 "Disable AVX512 optimizations" off) +option(PERNIX_ENABLE_X86_BMI2 "Build x86 BMI2 backend" ON) +option(PERNIX_ENABLE_X86_AVX2 "Build x86 AVX2 backend" ON) +option(PERNIX_ENABLE_X86_AVX512VBMI "Build x86 AVX512-VBMI backend" ON) +option(PERNIX_ENABLE_ARM64_NEON "Build arm64 NEON backend" ON) +option(PERNIX_ENABLE_ARM64_SVE2 "Build arm64 SVE2 backend" ON) option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" off) -set(PERNIX_SIMDE_PROVIDER "AUTO" CACHE STRING "SIMDe provider when PERNIX_USE_SIMDE is enabled (AUTO, PACKAGE, FETCH)") -set_property(CACHE PERNIX_SIMDE_PROVIDER PROPERTY STRINGS AUTO PACKAGE FETCH) -set(PERNIX_ARCH_BACKEND "AUTO" CACHE STRING "Pernix architecture backend (AUTO, FALLBACK, X86, ARM64_NEON, ARM64_SVE, ARM64_SVE2)") -set_property(CACHE PERNIX_ARCH_BACKEND PROPERTY STRINGS AUTO FALLBACK X86 ARM64_NEON ARM64_SVE ARM64_SVE2) option(PERNIX_ENABLE_FORTRAN_BINDINGS "Build Fortran bindings for pernix" off) -string(TOUPPER "${PERNIX_ARCH_BACKEND}" PERNIX_ARCH_BACKEND) -set(PERNIX_VALID_ARCH_BACKENDS AUTO FALLBACK X86 ARM64_NEON ARM64_SVE ARM64_SVE2) -if (NOT PERNIX_ARCH_BACKEND IN_LIST PERNIX_VALID_ARCH_BACKENDS) - message(FATAL_ERROR "Unsupported PERNIX_ARCH_BACKEND='${PERNIX_ARCH_BACKEND}'. Expected one of: ${PERNIX_VALID_ARCH_BACKENDS}") -endif () - -set(PERNIX_SELECTED_ARCH_BACKEND "${PERNIX_ARCH_BACKEND}") -if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "AUTO") - if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|i[3-6]86|i686)$") - set(PERNIX_SELECTED_ARCH_BACKEND "X86") - elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") - set(PERNIX_SELECTED_ARCH_BACKEND "ARM64_NEON") - else () - set(PERNIX_SELECTED_ARCH_BACKEND "FALLBACK") - endif () -endif () -message(STATUS "Pernix architecture backend: ${PERNIX_SELECTED_ARCH_BACKEND}") - -string(TOUPPER "${PERNIX_SIMDE_PROVIDER}" PERNIX_SIMDE_PROVIDER) -set(PERNIX_VALID_SIMDE_PROVIDERS AUTO PACKAGE FETCH) -if (NOT PERNIX_SIMDE_PROVIDER IN_LIST PERNIX_VALID_SIMDE_PROVIDERS) - message(FATAL_ERROR "Unsupported PERNIX_SIMDE_PROVIDER='${PERNIX_SIMDE_PROVIDER}'. Expected one of: ${PERNIX_VALID_SIMDE_PROVIDERS}") -endif () - list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") +#[===[ set(PERNIX_BUNDLE_SIMDE_FOR_INSTALL OFF) if (PERNIX_USE_SIMDE) if (PERNIX_SIMDE_PROVIDER STREQUAL "AUTO" OR PERNIX_SIMDE_PROVIDER STREQUAL "PACKAGE") @@ -75,6 +50,7 @@ if (PERNIX_USE_SIMDE) message(FATAL_ERROR "PERNIX_USE_SIMDE is enabled, but simde::simde was not found. Set PERNIX_SIMDE_PROVIDER=FETCH or install SIMDe's CMake package.") endif () endif () +]===] include(CTest) include(GitVersion) @@ -94,64 +70,74 @@ else () endif () message(STATUS "Pernix version: ${VERSION}, normalized to ${NORMALIZED_VERSION}") -if (MSVC) - message(FATAL_ERROR "MSVC compiler is not supported") -else () - include(CheckCXXCompilerFlag) - set(PERNIX_PRIVATE_COMPILE_OPTIONS) - foreach (PERNIX_CXX_FLAG - -Wall - -Wextra - -Wshadow - -Wfloat-equal - -Wold-style-cast - -Wconversion - -fstrict-aliasing - -Wno-ignored-attributes - ) - string(MAKE_C_IDENTIFIER "PERNIX_HAS_CXX_FLAG_${PERNIX_CXX_FLAG}" PERNIX_CXX_FLAG_VARIABLE) - check_cxx_compiler_flag("${PERNIX_CXX_FLAG}" "${PERNIX_CXX_FLAG_VARIABLE}") - if (${PERNIX_CXX_FLAG_VARIABLE}) - list(APPEND PERNIX_PRIVATE_COMPILE_OPTIONS "${PERNIX_CXX_FLAG}") - else () - message(STATUS "Compiler flag not supported: ${PERNIX_CXX_FLAG}") - endif () - endforeach () +if (NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|Intel") + message(FATAL_ERROR "Unsupported compiler: ${CMAKE_CXX_COMPILER_ID}") +endif () - if (PERNIX_ENABLE_LTO) - include(CheckIPOSupported) - check_ipo_supported(RESULT PERNIX_IPO_SUPPORTED OUTPUT PERNIX_IPO_ERROR) - if (NOT PERNIX_IPO_SUPPORTED) - message(FATAL_ERROR "PERNIX_ENABLE_LTO is enabled, but IPO/LTO is not supported: ${PERNIX_IPO_ERROR}") - endif () +include(CheckCXXCompilerFlag) +set(PERNIX_PRIVATE_COMPILE_OPTIONS) +foreach (PERNIX_CXX_FLAG + -Wall + -Wextra + -Wshadow + -Wfloat-equal + -Wold-style-cast + -Wconversion + -fstrict-aliasing + -Wno-ignored-attributes +) + string(MAKE_C_IDENTIFIER "PERNIX_HAS_CXX_FLAG_${PERNIX_CXX_FLAG}" PERNIX_CXX_FLAG_VARIABLE) + check_cxx_compiler_flag("${PERNIX_CXX_FLAG}" "${PERNIX_CXX_FLAG_VARIABLE}") + if (${PERNIX_CXX_FLAG_VARIABLE}) + list(APPEND PERNIX_PRIVATE_COMPILE_OPTIONS "${PERNIX_CXX_FLAG}") + else () + message(STATUS "Compiler flag not supported: ${PERNIX_CXX_FLAG}") + endif () +endforeach () - check_cxx_compiler_flag("-Wno-lto-type-mismatch" PERNIX_HAS_CXX_FLAG_WNO_LTO_TYPE_MISMATCH) - if (PERNIX_HAS_CXX_FLAG_WNO_LTO_TYPE_MISMATCH) - list(APPEND PERNIX_PRIVATE_COMPILE_OPTIONS "-Wno-lto-type-mismatch") - endif () +if (PERNIX_ENABLE_LTO) + include(CheckIPOSupported) + check_ipo_supported(RESULT PERNIX_IPO_SUPPORTED OUTPUT PERNIX_IPO_ERROR) + if (NOT PERNIX_IPO_SUPPORTED) + message(FATAL_ERROR "PERNIX_ENABLE_LTO is enabled, but IPO/LTO is not supported: ${PERNIX_IPO_ERROR}") + endif () - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - find_program(GCC_AR gcc-ar) - if (GCC_AR) - set(CMAKE_AR ${GCC_AR}) - endif () - find_program(GCC_RANLIB gcc-ranlib) - if (GCC_RANLIB) - set(CMAKE_RANLIB ${GCC_RANLIB}) - endif () - elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") - find_program(LLVM_AR llvm-ar) - if (LLVM_AR) - set(CMAKE_AR ${LLVM_AR}) - endif () - find_program(LLVM_RANLIB llvm-ranlib) - if (LLVM_RANLIB) - set(CMAKE_RANLIB ${LLVM_RANLIB}) - endif () + check_cxx_compiler_flag("-Wno-lto-type-mismatch" PERNIX_HAS_CXX_FLAG_WNO_LTO_TYPE_MISMATCH) + if (PERNIX_HAS_CXX_FLAG_WNO_LTO_TYPE_MISMATCH) + list(APPEND PERNIX_PRIVATE_COMPILE_OPTIONS "-Wno-lto-type-mismatch") + endif () + + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + find_program(GCC_AR gcc-ar) + if (GCC_AR) + set(CMAKE_AR ${GCC_AR}) + endif () + find_program(GCC_RANLIB gcc-ranlib) + if (GCC_RANLIB) + set(CMAKE_RANLIB ${GCC_RANLIB}) + endif () + elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + find_program(LLVM_AR llvm-ar) + if (LLVM_AR) + set(CMAKE_AR ${LLVM_AR}) + endif () + find_program(LLVM_RANLIB llvm-ranlib) + if (LLVM_RANLIB) + set(CMAKE_RANLIB ${LLVM_RANLIB}) endif () endif () endif () +set(PERNIX_TARGET_IS_X86 OFF) +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|i[3-6]86|i686)$") + set(PERNIX_TARGET_IS_X86 ON) +endif () + +set(PERNIX_TARGET_IS_ARM64 OFF) +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") + set(PERNIX_TARGET_IS_ARM64 ON) +endif () + add_subdirectory(src) if (PERNIX_ENABLE_FORTRAN_BINDINGS) diff --git a/cmake/pernixConfig.cmake.in b/cmake/pernixConfig.cmake.in new file mode 100644 index 0000000..fe3eb88 --- /dev/null +++ b/cmake/pernixConfig.cmake.in @@ -0,0 +1,18 @@ +@PACKAGE_INIT@ + +if (@PERNIX_USE_SIMDE@) + find_package(simde CONFIG QUIET) + if (NOT TARGET simde::simde AND @PERNIX_BUNDLE_SIMDE_FOR_INSTALL@) + add_library(simde::simde INTERFACE IMPORTED) + target_include_directories(simde::simde INTERFACE "${PACKAGE_PREFIX_DIR}/include") + endif () + if (NOT TARGET simde::simde) + set(pernix_FOUND FALSE) + set(pernix_NOT_FOUND_MESSAGE "pernix was built with SIMDe support, but simde::simde was not found. Install SIMDe's CMake package or use a Pernix package built with bundled SIMDe headers.") + return() + endif () +endif () + +include("${CMAKE_CURRENT_LIST_DIR}/pernixTargets.cmake") + +check_required_components(pernix) diff --git a/include/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h deleted file mode 100644 index 8e517fa..0000000 --- a/include/pernix/arm64/neon/common.h +++ /dev/null @@ -1,200 +0,0 @@ -#ifndef PERNIX_ARM64_NEON_COMMON_H -#define PERNIX_ARM64_NEON_COMMON_H - -#include - -#include - -namespace pernix::arm64::neon::internal { -struct float64x2x8_t { - float64x2_t val[8]; -}; - -static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t remaining_elements) { - const uint32_t tail_bits = remaining_elements * bit_width; - const uint32_t tail_bytes = (tail_bits + 7u) / 8u; - return tail_bytes; -} - -__always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(const int8x16_t& input) { - const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); - const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); - - return {{ - vmovl_s16(vget_low_s16(s16_lo)), - vmovl_s16(vget_high_s16(s16_lo)), - vmovl_s16(vget_low_s16(s16_hi)), - vmovl_s16(vget_high_s16(s16_hi)), - }}; -} - -__always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(const int16x8_t& input) { - return {{ - vmovl_s16(vget_low_s16(input)), - vmovl_s16(vget_high_s16(input)), - }}; -} - -__always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, const float32x4_t& scale) { - return {{ - vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), - }}; -} - -__always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t& input, const float32x4_t& scale) { - return {{ - vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), - }}; -} - -__always_inline float32x4_t neon_dequantize_epi32(const int32x4_t& input, const float32x4_t& scale) { - return vmulq_f32(vcvtq_f32_s32(input), scale); -} - -__always_inline float64x2_t neon_dequantize_epi32_f64(const int32x2_t& input, const float64x2_t& scale) { - return vmulq_f64(vcvtq_f64_s64(vmovl_s32(input)), scale); -} - -__always_inline float64x2x2_t neon_dequantize_epi32_f64(const int32x4_t& input, const float64x2_t& scale) { - return {{ - neon_dequantize_epi32_f64(vget_low_s32(input), scale), - neon_dequantize_epi32_f64(vget_high_s32(input), scale), - }}; -} - -__always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t& input, const float64x2_t& scale) { - const float64x2x2_t dequantized_low = neon_dequantize_epi32_f64(input.val[0], scale); - const float64x2x2_t dequantized_high = neon_dequantize_epi32_f64(input.val[1], scale); - - return {{ - dequantized_low.val[0], - dequantized_low.val[1], - dequantized_high.val[0], - dequantized_high.val[1], - }}; -} - -__always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input, const float64x2_t& scale) { - const float64x2x2_t dequantized0 = neon_dequantize_epi32_f64(input.val[0], scale); - const float64x2x2_t dequantized1 = neon_dequantize_epi32_f64(input.val[1], scale); - const float64x2x2_t dequantized2 = neon_dequantize_epi32_f64(input.val[2], scale); - const float64x2x2_t dequantized3 = neon_dequantize_epi32_f64(input.val[3], scale); - - return {{ - dequantized0.val[0], - dequantized0.val[1], - dequantized1.val[0], - dequantized1.val[1], - dequantized2.val[0], - dequantized2.val[1], - dequantized3.val[0], - dequantized3.val[1], - }}; -} - -__always_inline uint8x16_t neon_load_tail_elements_int8(const uint8_t* input, const uint32_t tail_bytes_count) { - uint8_t buffer[16] = {0}; - std::memcpy(buffer, input, tail_bytes_count); - return vld1q_u8(buffer); -} - -__always_inline uint16x8_t neon_load_tail_elements_int16(const uint8_t* input, const uint32_t tail_bytes_count) { - uint16_t buffer[8] = {0}; - std::memcpy(buffer, input, tail_bytes_count); - return vld1q_u16(buffer); -} - -__always_inline uint32x4_t neon_load_tail_elements_int32(const uint8_t* input, const uint32_t tail_bytes_count) { - uint32_t buffer[4] = {0}; - std::memcpy(buffer, input, tail_bytes_count); - return vld1q_u32(buffer); -} - -__always_inline float32x4_t neon_load_tail_elements_f32(const uint8_t* input, const uint32_t tail_elements) { - float32_t buffer[4] = {0.0f}; - std::memcpy(buffer, input, tail_elements * sizeof(float32_t)); - return vld1q_f32(buffer); -} - -__always_inline float64x2_t neon_load_tail_elements_f64(const uint8_t* input, const uint32_t tail_elements) { - float64_t buffer[2] = {0.0}; - std::memcpy(buffer, input, tail_elements * sizeof(float64_t)); - return vld1q_f64(buffer); -} - -__always_inline void neon_store_tail_elements_int8(uint8_t* output, const uint8x16x4_t& data, const uint32_t tail_elements) { - uint8_t buffer[16 * 4]; - for (uint32_t i = 0; i < 4; ++i) { - vst1q_u8(buffer + i * 16, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(uint8_t)); -} - -__always_inline void neon_store_tail_elements_int16(uint16_t* output, const uint16x8x4_t& data, const uint32_t tail_elements) { - uint16_t buffer[8 * 4]; - for (uint32_t i = 0; i < 4; ++i) { - vst1q_u16(buffer + i * 8, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(uint16_t)); -} - -__always_inline void neon_store_tail_elements_int32(uint32_t* output, const uint32x4x4_t& data, const uint32_t tail_elements) { - uint32_t buffer[4 * 4]; - for (uint32_t i = 0; i < 4; ++i) { - vst1q_u32(buffer + i * 4, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(uint32_t)); -} - -__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x4_t& data, const uint32_t tail_elements) { - float32_t buffer[16 * 4]; - for (uint32_t i = 0; i < 4; ++i) { - vst1q_f32(buffer + i * 4, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); -} - -__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x2_t& data, const uint32_t tail_elements) { - float32_t buffer[8 * 2]; - for (uint32_t i = 0; i < 2; ++i) { - vst1q_f32(buffer + i * 4, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); -} - -__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4_t& data, const uint32_t tail_elements) { - float32_t buffer[4]; - vst1q_f32(buffer, data); - std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); -} - -__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x4_t& data, const uint32_t tail_elements) { - float64_t buffer[2 * 4]; - for (uint32_t i = 0; i < 4; ++i) { - vst1q_f64(buffer + i * 2, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); -} - -__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x2_t& data, const uint32_t tail_elements) { - float64_t buffer[2 * 2]; - for (uint32_t i = 0; i < 2; ++i) { - vst1q_f64(buffer + i * 2, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); -} - -__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x8_t& data, const uint32_t tail_elements) { - float64_t buffer[2 * 8]; - for (uint32_t i = 0; i < 8; ++i) { - vst1q_f64(buffer + i * 2, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); -} -} // namespace pernix::arm64::neon::internal - -#endif // PERNIX_ARM64_NEON_COMMON_H diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h deleted file mode 100644 index 6ac0e20..0000000 --- a/include/pernix/arm64/neon/unpacking.h +++ /dev/null @@ -1,100 +0,0 @@ -#ifndef PERNIX_ARM64_NEON_UNPACKING_H -#define PERNIX_ARM64_NEON_UNPACKING_H - -#include -#include - -using namespace pernix::arm64::neon::internal; - -namespace pernix::arm64::neon::internal::b128 { -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t& input) { - if constexpr (BIT_WIDTH == 8) { - return vreinterpretq_s8_u8(input); - } else if constexpr (BIT_WIDTH == 1) { - using tables = table_unpacking; - - const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); - const uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); - - return vreinterpretq_s8_u8(vandq_u8(shifted, vdupq_n_u8(1))); - } else { - using tables = table_unpacking; - - const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); - - uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); - - if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { - const uint8x16_t permuted2_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute2.data())); - - shifted = vorrq_u8(shifted, vshlq_u8(permuted2_u8, vld1q_s8(tables::shift2.data()))); - } - - constexpr int shift = 8 - BIT_WIDTH; - shifted = vshlq_n_u8(shifted, shift); - - if constexpr (SIGN_VALUES) { - return vshlq_s8(vreinterpretq_s8_u8(shifted), vdupq_n_s8(-shift)); - } else { - return vreinterpretq_s8_u8(vshlq_u8(shifted, vdupq_n_s8(-shift))); - } - } -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t& input) { - if constexpr (BIT_WIDTH == 16) { - return vreinterpretq_s16_u16(input); - } else { - using tables = table_unpacking; - - const uint8x16_t input_u8 = vreinterpretq_u8_u16(input); - - const uint8x16_t permuted1_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute1.data())); - - uint16x8_t shifted = vshlq_u16(vreinterpretq_u16_u8(permuted1_u8), vld1q_s16(tables::shift1.data())); - - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const uint8x16_t permuted2_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute2.data())); - - const uint16x8_t shifted2 = vshlq_u16(vreinterpretq_u16_u8(permuted2_u8), vld1q_s16(tables::shift2.data())); - - shifted = vorrq_u16(shifted, shifted2); - } - - constexpr int shift = 16 - BIT_WIDTH; - shifted = vshlq_n_u16(shifted, shift); - - if constexpr (SIGN_VALUES) { - return vshlq_s16(vreinterpretq_s16_u16(shifted), vdupq_n_s16(-shift)); - } else { - return vreinterpretq_s16_u16(vshlq_u16(shifted, vdupq_n_s16(-shift))); - } - } -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline int32x4_t neon_unpack_epi32_17to24(const uint32x4_t& input) { - using tables = table_unpacking; - - const uint8x16_t input_8 = vreinterpretq_u8_u32(input); - - const uint8x16_t permuted_u8 = vqtbl1q_u8(input_8, vld1q_u8(tables::permute.data())); - - const uint32x4_t value = vshlq_u32(vreinterpretq_u32_u8(permuted_u8), vld1q_s32(tables::shift.data())); - - if constexpr (SIGN_VALUES) { - constexpr int sign_shift = 32 - BIT_WIDTH; - return vshrq_n_s32(vreinterpretq_s32_u32(vshlq_n_u32(value, sign_shift)), sign_shift); - } else { - constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1u; - return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); - } -} -} // namespace pernix::arm64::neon::internal::b128 - -#endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/arm64/sve/compression.h b/include/pernix/arm64/sve/compression.h deleted file mode 100644 index cf83ce0..0000000 --- a/include/pernix/arm64/sve/compression.h +++ /dev/null @@ -1,141 +0,0 @@ -#ifndef PERNIX_ARM64_SVE_COMPRESSION_H -#define PERNIX_ARM64_SVE_COMPRESSION_H - -#include -#include - -#include -#include - -namespace pernix::arm64::sve { -namespace internal { -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_compress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_compress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_compress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_compress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_compress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_compress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} -} // namespace internal - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_compress_block(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::sve_compress_block_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::sve_compress_block_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::sve_compress_block_17to24(input, scale, output); - } - return 0; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_compress_block(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::sve_compress_block_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::sve_compress_block_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::sve_compress_block_17to24(input, scale, output); - } - return 0; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_compress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - sve_compress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_compress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - sve_compress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - return 0; -} - -#ifdef __cplusplus -extern "C" { -#endif - -int sve_compress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - - -int sve_compress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output); - -int sve_compress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -int sve_compress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output, uint32_t blocks); - -#ifdef __cplusplus -} -#endif -} // namespace pernix::arm64::sve - -#endif // PERNIX_ARM64_SVE_COMPRESSION_H diff --git a/include/pernix/arm64/sve/decompression.h b/include/pernix/arm64/sve/decompression.h deleted file mode 100644 index 052a3e4..0000000 --- a/include/pernix/arm64/sve/decompression.h +++ /dev/null @@ -1,141 +0,0 @@ -#ifndef PERNIX_ARM64_SVE_DECOMPRESSION_H -#define PERNIX_ARM64_SVE_DECOMPRESSION_H - -#include -#include - -#include -#include - -namespace pernix::arm64::sve { -namespace internal { -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} -} // namespace internal - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_decompress_block(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::sve_decompress_block_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::sve_decompress_block_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::sve_decompress_block_17to24(input, scale, output); - } - return 0; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve_decompress_block(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::sve_decompress_block_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::sve_decompress_block_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::sve_decompress_block_17to24(input, scale, output); - } - return 0; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - sve_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve_decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - sve_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - return 0; -} - -#ifdef __cplusplus -extern "C" { -#endif - -int sve_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - - -int sve_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output); - -int sve_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -int sve_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output, uint32_t blocks); - -#ifdef __cplusplus -} -#endif -} // namespace pernix::arm64::sve - -#endif // PERNIX_ARM64_SVE_DECOMPRESSION_H diff --git a/include/pernix/arm64/sve/packing.h b/include/pernix/arm64/sve/packing.h deleted file mode 100644 index ab57b4f..0000000 --- a/include/pernix/arm64/sve/packing.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef PERNIX_ARM64_SVE_PACKING_H -#define PERNIX_ARM64_SVE_PACKING_H - -#include - -namespace pernix::arm64::sve::internal { -} // namespace pernix::arm64::sve::internal - -#endif // PERNIX_ARM64_SVE_PACKING_H diff --git a/include/pernix/arm64/sve/unpacking.h b/include/pernix/arm64/sve/unpacking.h deleted file mode 100644 index 2565ab7..0000000 --- a/include/pernix/arm64/sve/unpacking.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef PERNIX_ARM64_SVE_UNPACKING_H -#define PERNIX_ARM64_SVE_UNPACKING_H - -#include - -namespace pernix::arm64::sve::internal { -} // namespace pernix::arm64::sve::internal - -#endif // PERNIX_ARM64_SVE_UNPACKING_H diff --git a/include/pernix/arm64/sve2/tables.h b/include/pernix/arm64/sve2/tables.h deleted file mode 100644 index 897fa9b..0000000 --- a/include/pernix/arm64/sve2/tables.h +++ /dev/null @@ -1,118 +0,0 @@ -#ifndef PERNIX_ARM64_SVE2_TABLES_H -#define PERNIX_ARM64_SVE2_TABLES_H - -#include - -#include - -namespace pernix::arm64::sve2::internal { -template -struct table_unpacking { - static constexpr uint8_t bit_width = BIT_WIDTH; - - static svbool_t pg_b8() { return svptrue_b8(); } - - static svbool_t pg_b16() { return svptrue_b16(); } - - static svbool_t pg_b32() { return svptrue_b32(); } -}; - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -struct table_unpacking { - static constexpr uint8_t bit_width = BIT_WIDTH; - - static svuint8_t permute() { - const svbool_t pg = svptrue_b8(); - return svlsr_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 3); - } - - static svuint8_t spill_permute() { - const svbool_t pg = svptrue_b8(); - return svadd_n_u8_x(pg, permute(), 1); - } - - static svuint8_t shift() { - const svbool_t pg = svptrue_b8(); - return svand_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 7); - } - - static svuint8_t spill_shift() { - const svbool_t pg = svptrue_b8(); - return svsub_u8_x(pg, svdup_n_u8(8), shift()); - } -}; - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -struct table_unpacking { - static constexpr uint8_t bit_width = BIT_WIDTH; - - static svuint8_t permute() { - const svbool_t pg = svptrue_b8(); - const svuint8_t lane = svindex_u8(0, 1); - const svuint8_t elem = svlsr_n_u8_x(pg, lane, 1); - const svuint8_t byte = svand_n_u8_x(pg, lane, 1); - - svuint8_t first; - if constexpr (BIT_WIDTH == 16) { - first = svlsl_n_u8_x(pg, elem, 1); - } else { - constexpr uint8_t extra_bits = BIT_WIDTH - 8u; - const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); - const svuint8_t low = svlsr_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), 3); - first = svadd_u8_x(pg, elem, svadd_u8_x(pg, high, low)); - } - - return svadd_u8_x(pg, first, byte); - } - - static svuint8_t spill_permute() { - const svbool_t pg = svptrue_b8(); - return svadd_n_u8_x(pg, permute(), 2); - } - - static svuint16_t shift() { - const svbool_t pg = svptrue_b16(); - return svand_n_u16_x(pg, svmul_n_u16_x(pg, svindex_u16(0, 1), BIT_WIDTH), 7); - } - - static svuint16_t spill_shift() { - const svbool_t pg = svptrue_b16(); - const svuint16_t bit_shift = shift(); - const svuint16_t spill = svsub_u16_x(pg, svdup_n_u16(16), bit_shift); - return svsel_u16(svcmpgt_n_u16(pg, bit_shift, 16u - BIT_WIDTH), spill, svdup_n_u16(16)); - } -}; - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && START_BIT_OFFSET < 8) -struct table_unpacking { - static constexpr uint8_t bit_width = BIT_WIDTH; - - static svuint8_t permute() { - const svbool_t pg = svptrue_b8(); - const svuint8_t lane = svindex_u8(0, 1); - const svuint8_t elem = svlsr_n_u8_x(pg, lane, 2); - const svuint8_t byte = svand_n_u8_x(pg, lane, 3); - - svuint8_t first = svmul_n_u8_x(pg, elem, BIT_WIDTH / 8u); - if constexpr (BIT_WIDTH % 8u != 0) { - constexpr uint8_t extra_bits = BIT_WIDTH % 8u; - const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); - const svuint8_t low_bits = - svadd_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), START_BIT_OFFSET); - first = svadd_u8_x(pg, first, svadd_u8_x(pg, high, svlsr_n_u8_x(pg, low_bits, 3))); - } - - return svadd_u8_x(pg, first, byte); - } - - static svuint32_t shift() { - const svbool_t pg = svptrue_b32(); - return svand_n_u32_x(pg, svadd_n_u32_x(pg, svmul_n_u32_x(pg, svindex_u32(0, 1), BIT_WIDTH), START_BIT_OFFSET), 7); - } -}; -} // namespace pernix::arm64::sve2::internal - -#endif // PERNIX_ARM64_SVE2_TABLES_H diff --git a/include/pernix/arm64/sve2/unpacking.h b/include/pernix/arm64/sve2/unpacking.h deleted file mode 100644 index 326901f..0000000 --- a/include/pernix/arm64/sve2/unpacking.h +++ /dev/null @@ -1,92 +0,0 @@ -#ifndef PERNIX_ARM64_SVE2_UNPACKING_H -#define PERNIX_ARM64_SVE2_UNPACKING_H - -#include - -#include "tables.h" - -namespace pernix::arm64::sve2::internal { -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline svint8_t sve2_unpack_epi8_1to8(const svuint8_t input, const svuint8_t permute, const svuint8_t shift, - const svuint8_t spill_permute, const svuint8_t spill_shift) { - if constexpr (BIT_WIDTH == 8) { - return svreinterpret_s8(input); - } else { - const svbool_t pg = svptrue_b8(); - - const svuint8_t permuted = svtbl_u8(input, permute); - svuint8_t unpacked = svlsr_u8_x(pg, permuted, shift); - - if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { - const svuint8_t spill_permuted_values = svtbl_u8(input, spill_permute); - const svuint8_t spill_shifted = svlsl_u8_x(pg, spill_permuted_values, spill_shift); - unpacked = svorr_u8_x(pg, unpacked, spill_shifted); - } - - if constexpr (BIT_WIDTH == 1) { - unpacked = svand_n_u8_x(pg, unpacked, 1); - return svreinterpret_s8(unpacked); - } else { - constexpr int sign_shift = 8 - BIT_WIDTH; - - unpacked = svlsl_n_u8_x(pg, unpacked, sign_shift); - - if constexpr (SIGN_VALUES) { - return svasr_n_s8_x(pg, svreinterpret_s8_u8(unpacked), sign_shift); - } else { - return svreinterpret_s8_u8(svlsr_n_u8_x(pg, unpacked, sign_shift)); - } - } - } -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline svint16_t sve2_unpack_epi16_9to16(const svuint16_t input, const svuint8_t permute, const svuint16_t shift, - const svuint8_t spill_permute, const svuint16_t spill_shift) { - if constexpr (BIT_WIDTH == 16) { - return svreinterpret_s16(input); - } else { - const svbool_t pg = svptrue_b16(); - - const svuint8_t permuted = svtbl_u8(svreinterpret_u8_u16(input), permute); - svuint16_t shifted = svlsr_u16_x(pg, svreinterpret_u16_u8(permuted), shift); - - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const svuint8_t spill_permuted_values = svtbl_u8(svreinterpret_u8_u16(input), spill_permute); - const svuint16_t spill_shifted = svlsl_u16_x(pg, svreinterpret_u16_u8(spill_permuted_values), spill_shift); - shifted = svorr_u16_x(pg, shifted, spill_shifted); - } - - constexpr int sign_shift = 16 - BIT_WIDTH; - shifted = svlsl_n_u16_x(pg, shifted, sign_shift); - - if constexpr (SIGN_VALUES) { - return svasr_n_s16_x(pg, svreinterpret_s16_u16(shifted), sign_shift); - } else { - return svreinterpret_s16_u16(svlsr_n_u16_x(pg, shifted, sign_shift)); - } - } -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline svint32_t sve2_unpack_epi32_17to24(const svuint8_t input) { - using table = table_unpacking; - - const svbool_t pg = svptrue_b32(); - const svuint8_t permuted = svtbl_u8(input, table::permute()); - const svuint32_t unpacked = svlsr_u32_x(pg, svreinterpret_u32_u8(permuted), table::shift()); - - if constexpr (SIGN_VALUES) { - constexpr int sign_shift = 32 - BIT_WIDTH; - return svasr_n_s32_x(pg, svreinterpret_s32_u32(svlsl_n_u32_x(pg, unpacked, sign_shift)), sign_shift); - } else { - constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1u; - return svreinterpret_s32_u32(svand_n_u32_x(pg, unpacked, mask)); - } -} -} // namespace pernix::arm64::sve2::internal - -#endif // PERNIX_ARM64_SVE2_UNPACKING_H diff --git a/include/pernix/compat.h b/include/pernix/compat.h new file mode 100644 index 0000000..42544ee --- /dev/null +++ b/include/pernix/compat.h @@ -0,0 +1,24 @@ +#ifndef PERNIX_COMPAT_H +#define PERNIX_COMPAT_H + +#ifndef __always_inline +#if defined(__GNUC__) || defined(__clang__) +#define __always_inline inline __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define __always_inline __forceinline +#else +#define __always_inline inline +#endif +#endif + +#if defined(_WIN32) && defined(PERNIX_SHARED) +#if defined(PERNIX_BUILD_LIB) +#define PERNIX_API __declspec(dllexport) +#else +#define PERNIX_API __declspec(dllimport) +#endif +#else +#define PERNIX_API +#endif + +#endif //PERNIX_COMPAT_H diff --git a/include/pernix/detection.h b/include/pernix/detection.h deleted file mode 100644 index fa9cd44..0000000 --- a/include/pernix/detection.h +++ /dev/null @@ -1,87 +0,0 @@ -#ifndef PERNIX_HELPER_H -#define PERNIX_HELPER_H - -#include - -// Internal capability tiers used to derive the public PERNIX_* feature macros. -#define PERNIX_MACHINE_ID_GENERIC 0 -#define PERNIX_MACHINE_ID_V2 1 -#define PERNIX_MACHINE_ID_V3 2 -#define PERNIX_MACHINE_ID_V4 3 -#define PERNIX_MACHINE_ID_V4_VBMI 4 - -#if defined(PERNIX_BACKEND_ARM64_NEON) -#define PERNIX_ARM64_NEON_ENABLED -#endif - -#if defined(PERNIX_BACKEND_ARM64_SVE) -#define PERNIX_ARM64_SVE_ENABLED -#endif - -#if defined(PERNIX_BACKEND_ARM64_SVE2) -#define PERNIX_ARM64_SVE2_ENABLED -#endif - -#if defined(PERNIX_BACKEND_X86) -// Map the compiler's enabled ISA set to the highest supported Pernix target level. -#if (__SSE3__ && __SSE4_1__ && __SSE4_2__) -#if (__AVX__ && __AVX2__ && __FMA__ && __BMI__ && __BMI2__) -#if (__AVX512BW__ && __AVX512CD__ && __AVX512DQ__ && __AVX512F__ && __AVX512VL__) -#if (__AVX512VBMI__) -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_V4_VBMI -#else -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_V4 -#endif - -#else -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_V3 -#endif - -#else -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_V2 -#endif - -#else -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_GENERIC -#endif - -#else -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_GENERIC -#endif - -// Feature-selection macros consumed by the public headers. -#if (PERNIX_MACHINE_ID >= PERNIX_MACHINE_ID_V2) -#define PERNIX_SSE_ENABLED -#endif -#if (PERNIX_MACHINE_ID >= PERNIX_MACHINE_ID_V3) -#define PERNIX_AVX2_ENABLED -#define PERNIX_BMI2_ENABLED -#endif -#if (PERNIX_MACHINE_ID >= PERNIX_MACHINE_ID_V4) -#define PERNIX_AVX512_ENABLED -#endif -#if (PERNIX_MACHINE_ID >= PERNIX_MACHINE_ID_V4_VBMI) -#define PERNIX_AVX512_VBMI_ENABLED -#endif - -#if defined(PERNIX_USE_SIMDE) && defined(PERNIX_BACKEND_X86) -#define PERNIX_SSE_ENABLED -#define PERNIX_AVX2_ENABLED -#define PERNIX_BMI2_ENABLED -#define PERNIX_AVX512_ENABLED -#define PERNIX_AVX512_VBMI_ENABLED -#endif - -// Allow build systems or tests to force lower-tier implementations. -#ifdef PERNIX_DISABLE_AVX512 -#undef PERNIX_AVX512_ENABLED -#undef PERNIX_AVX512_VBMI_ENABLED -#endif -#ifdef PERNIX_DISABLE_AVX2 -#undef PERNIX_AVX2_ENABLED -#endif -#ifdef PERNIX_DISABLE_BMI2 -#undef PERNIX_BMI2_ENABLED -#endif - -#endif // PERNIX_HELPER_H diff --git a/include/pernix/fallback/decompression.h b/include/pernix/fallback/decompression.h deleted file mode 100644 index c4714e3..0000000 --- a/include/pernix/fallback/decompression.h +++ /dev/null @@ -1,286 +0,0 @@ -#ifndef PERNIX_FALLBACK_DECOMPRESSION_H -#define PERNIX_FALLBACK_DECOMPRESSION_H - -#include - -#include -#include -#include -#include - -namespace pernix { -namespace internal { -/** - * @brief Dequantize a single int32_t value to float using the provided scale. - * - * @param input input int32_t value to be dequantized. - * @param scale scaling factor used during quantization. - * @return float dequantized float value. - */ -__always_inline float dequantize_epi32(const int32_t input, const float scale) { - return static_cast(input) * scale; -} - -/** - * @brief Dequantize a single int64_t value to double using the provided scale. - * - * @param input input int64_t value to be dequantized. - * @param scale scaling factor used during quantization. - * @return double_t dequantized double value. - */ -__always_inline double_t dequantize_epi64(const int64_t input, const double_t scale) { - return static_cast(input) * scale; -} - -/** - * @brief Sign-extend a packed integer value stored in the low bits of a 32-bit word. - * - * @tparam BIT_WIDTH number of significant bits in the encoded value. - * @param value unsigned packed value. - * @return int32_t sign-extended value. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline auto sign_extend(const uint32_t value) -> int32_t { - if constexpr (BIT_WIDTH == 1) { - return static_cast(value & 1U); - } - - constexpr uint32_t sign_bit = uint32_t{1} << (BIT_WIDTH - 1); - constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1; - const uint32_t masked = value & mask; - return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); -} - -/** - * @brief Unpack bit-packed values from a typed input span into signed 32-bit integers. - * - * @tparam T unsigned integer type used to read the source buffer. - * @tparam BIT_WIDTH bit width per packed value. - * @tparam SIGN_VALUES whether to sign-extend unpacked values. - * @param input pointer to the typed packed input buffer. - * @param bit_offset starting bit offset in the first input word. - * @param elements number of values to unpack. - * @return std::vector unpacked values. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) -__always_inline auto unpack_epi32_fallback_inner(const uint8_t* __restrict__ input, const uint8_t bit_offset, const std::size_t elements) - -> std::vector { - constexpr uint32_t bits_in_type = sizeof(T) * 8; - constexpr uint32_t bitmask = BIT_WIDTH == bits_in_type ? std::numeric_limits::max() : (1U << BIT_WIDTH) - 1U; - - std::vector output(elements); - - std::size_t idx = 0; - uint8_t bits_in_buffer = 8 - bit_offset; - uint64_t buffer = static_cast(input[idx++]) >> bit_offset; - -#pragma GCC unroll 64 - for (uint32_t i = 0; i < elements; i++) { - while (BIT_WIDTH > bits_in_buffer) { - const auto next_value = static_cast(input[idx++]) << bits_in_buffer; - buffer |= next_value; - bits_in_buffer += 8; - } - - const uint32_t raw_value = static_cast(buffer & bitmask); - if constexpr (SIGN_VALUES) { - output[i] = sign_extend(raw_value); - } else { - output[i] = static_cast(raw_value); - } - - buffer >>= BIT_WIDTH; - bits_in_buffer -= BIT_WIDTH; - } - - return output; -} - -/** - * @brief Unpack packed int32_t values from the input buffer using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the packed data. - * @param elements number of elements to unpack. - * @return std::vector unpacked int32_t values. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline auto unpack_epi32_fallback(const uint8_t* __restrict__ input, const std::size_t elements) -> std::vector { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else { - return unpack_epi32_fallback_inner(input, 0, elements); - } -} -} // namespace internal - -/** - * @brief Decompress a single 512\-bit block using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_block_fallback(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - const std::vector block_values = internal::unpack_epi32_fallback(input, elements_per_block); - -#pragma GCC unroll 512 - for (uint32_t i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi32(block_values[i], scale); - } - - return 0; -} - -/** - * @brief Decompress a single block to double values using the fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_block_fallback(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - const std::vector block_values = internal::unpack_epi32_fallback(input, elements_per_block); - -#pragma GCC unroll 512 - for (uint32_t i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi64(block_values[i], scale); - } - - return 0; -} - -/** - * @brief Decompress multiple 512\-bit blocks using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int decompress_blocks_fallback(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; -} - -/** - * @brief Decompress multiple blocks to double values using the fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @param blocks number of blocks to decompress. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_blocks_fallback(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; -} -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Decompress a single 512-bit block using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - */ -int decompress_block_fallback(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single 512-bit block using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - */ -int decompress_block_fallback_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -/** - * @brief Decompress multiple 512-bit blocks using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - */ -int decompress_blocks_fallback(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Decompress multiple 512-bit blocks using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - */ -int decompress_blocks_fallback_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - -#endif // PERNIX_FALLBACK_DECOMPRESSION_H diff --git a/include/pernix/pernix.h b/include/pernix/pernix.h index 4998a60..7a64af6 100644 --- a/include/pernix/pernix.h +++ b/include/pernix/pernix.h @@ -1,582 +1,58 @@ #ifndef PERNIX_H #define PERNIX_H -#include +#include +#include +#include -// Include architecture-specific headers based on detected capabilities -// AVX2 -#if defined(PERNIX_BACKEND_X86) && defined(PERNIX_AVX2_ENABLED) -#include -#include - -// BMI2: Needs AVX2 as well -#ifdef PERNIX_BMI2_ENABLED -#include -#include -#endif // PERNIX_BMI2_ENABLED - -// AVX512 VBMI: Needs AVX2 as well -#ifdef PERNIX_AVX512_VBMI_ENABLED -#include -#include -#endif // PERNIX_AVX512_VBMI_ENABLED - -#endif // PERNIX_BACKEND_X86 && PERNIX_AVX2_ENABLED - -#ifdef PERNIX_BACKEND_ARM64_NEON -#include -#include -#endif - -#ifdef PERNIX_BACKEND_ARM64_SVE -#include -#include -#endif - -#ifdef PERNIX_BACKEND_ARM64_SVE2 -#include -#include -#endif - -// Fallback (non-SIMD) implementations -#include -#include - -namespace pernix { -/** - * @brief Compress a single block of floating-point data into a bit-packed format using the specified bit width and scale. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * - * @return int status code (0 for success). - * - * @note This function will dispatch to the best available implementation based on detected CPU features at compile time. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single block of double-precision values into a bit-packed representation. - * - * @tparam BIT_WIDTH bit width per quantized value (1 to 24). - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the input block. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress multiple blocks of single-precision values. - * - * @tparam BIT_WIDTH bit width per quantized value (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (must be a multiple of 32). - * - * @param input pointer to the first input value. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @param blocks number of consecutive blocks to process. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, uint32_t blocks); - -/** - * @brief Compress multiple blocks of double-precision values. - * - * @tparam BIT_WIDTH bit width per quantized value (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (must be a multiple of 32). - * - * @param input pointer to the first input value. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @param blocks number of consecutive blocks to process. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, uint32_t blocks); - -/** - * @brief Decompress a single block of packed values into single-precision values. - * - * @tparam BIT_WIDTH bit width per packed value (1 to 24). - * @tparam SIGN_VALUES true for signed values, false for unsigned values. - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the packed input block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single block of packed values into double-precision values. - * - * @tparam BIT_WIDTH bit width per packed value (1 to 24). - * @tparam SIGN_VALUES true for signed values, false for unsigned values. - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the packed input block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -/** - * @brief Decompress multiple blocks of packed values into single-precision values. - * - * @tparam BIT_WIDTH bit width per packed value (1 to 24). - * @tparam SIGN_VALUES true for signed values, false for unsigned values. - * @tparam BLOCK_SIZE size of each block in bytes (must be a multiple of 32). - * - * @param input pointer to the first packed block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @param blocks number of consecutive blocks to process. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); - -/** - * @brief Decompress multiple blocks of packed values into double-precision values. - * - * @tparam BIT_WIDTH bit width per packed value (1 to 24). - * @tparam SIGN_VALUES true for signed values, false for unsigned values. - * @tparam BLOCK_SIZE size of each block in bytes (must be a multiple of 32). - * - * @param input pointer to the first packed block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @param blocks number of consecutive blocks to process. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, uint32_t blocks); - -// Use the best available implementation based on detected CPU features at compile time. -#if defined(PERNIX_BACKEND_X86) && defined(PERNIX_AVX2_ENABLED) -#ifdef PERNIX_AVX512_VBMI_ENABLED -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return mm512_compress_block_avx512vbmi(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return mm512_compress_block_avx512vbmi(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return mm512_compress_blocks_avx512vbmi(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return mm512_compress_blocks_avx512vbmi(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return mm512_decompress_block_avx512vbmi(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return mm512_decompress_block_avx512vbmi(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - return mm512_decompress_blocks_avx512vbmi(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - return mm512_decompress_blocks_avx512vbmi(input, scale, output, blocks); -} -#else -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return mm256_compress_block_avx2(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return mm256_compress_block_avx2(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return mm256_compress_blocks_avx2(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return mm256_compress_blocks_avx2(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return mm256_decompress_block_avx2(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return mm256_decompress_block_avx2(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - return mm256_decompress_blocks_avx2(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - return mm256_decompress_blocks_avx2(input, scale, output, blocks); -} +#if defined(__cplusplus) +extern "C" { #endif -#elif defined(PERNIX_BACKEND_ARM64_NEON) -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return neon_compress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return neon_compress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return neon_compress_blocks(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return neon_compress_blocks(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return neon_decompress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return neon_decompress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - return neon_decompress_blocks(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - return neon_decompress_blocks(input, scale, output, blocks); -} -#elif defined(PERNIX_BACKEND_ARM64_SVE) -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return sve_compress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return sve_compress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return sve_compress_blocks(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return sve_compress_blocks(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return sve_decompress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return sve_decompress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - return sve_decompress_blocks(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - return sve_decompress_blocks(input, scale, output, blocks); -} -#elif defined(PERNIX_BACKEND_ARM64_SVE2) -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return sve2_compress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return sve2_compress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return sve2_compress_blocks(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return sve2_compress_blocks(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return sve2_decompress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return sve2_decompress_block(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - return sve2_decompress_blocks(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - return sve2_decompress_blocks(input, scale, output, blocks); -} -#else -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return compress_block_fallback(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return compress_block_fallback(input, scale, output); -} -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return compress_blocks_fallback(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return compress_blocks_fallback(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return decompress_block_fallback(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return decompress_block_fallback(input, scale, output); -} +typedef enum pernix_status { + PERNIX_STATUS_OK = 0, + PERNIX_STATUS_INVALID_ARGUMENT = -1, + PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH = -2, + PERNIX_STATUS_UNSUPPORTED_BACKEND = -3, + PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE = -4 +} pernix_status; -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - return decompress_blocks_fallback(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - return decompress_blocks_fallback(input, scale, output, blocks); -} -#endif -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif +typedef enum pernix_backend { + PERNIX_BACKEND_AUTO = 0, + PERNIX_BACKEND_FALLBACK = 1, + PERNIX_BACKEND_X86_AVX2 = 2, + PERNIX_BACKEND_X86_BMI2 = 3, + PERNIX_BACKEND_X86_AVX512_VBMI = 4, + PERNIX_BACKEND_ARM64_NEON = 5, + PERNIX_BACKEND_ARM64_SVE = 6 +} pernix_backend; -/** - * @brief C ABI wrapper for compressing one single-precision block. - * - * @param bit_width bit width per quantized value (8 to 16). - * @param input pointer to the input block. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int compress_block(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); +PERNIX_API pernix_status pernix_compress_block_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + float scale, void* output); -/** - * @brief C ABI wrapper for compressing one double-precision block. - * - * @param bit_width bit width per quantized value (8 to 16). - * @param input pointer to the input block. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int compress_block_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); +PERNIX_API pernix_status pernix_compress_blocks_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + float scale, void* output, uint32_t blocks); -/** - * @brief C ABI wrapper for compressing multiple single-precision blocks. - * - * @param bit_width bit width per quantized value (8 to 16). - * @param input pointer to the first input value. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @param blocks number of consecutive blocks to process. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int compress_blocks(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, uint32_t blocks); +PERNIX_API pernix_status pernix_decompress_block_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + float scale, void* output, bool sign_values); -/** - * @brief C ABI wrapper for compressing multiple double-precision blocks. - * - * @param bit_width bit width per quantized value (8 to 16). - * @param input pointer to the first input value. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @param blocks number of consecutive blocks to process. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int compress_blocks_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); +PERNIX_API pernix_status pernix_decompress_blocks_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + float scale, void* output, uint32_t blocks, bool sign_values); -/** - * @brief C ABI wrapper for decompressing one single-precision block. - * - * @param bit_width bit width per packed value (1 to 24). - * @param input pointer to the packed input block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); +PERNIX_API pernix_status pernix_compress_block_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + double scale, void* output); -/** - * @brief C ABI wrapper for decompressing one double-precision block. - * - * @param bit_width bit width per packed value (1 to 24). - * @param input pointer to the packed input block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); +PERNIX_API pernix_status pernix_compress_blocks_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + double scale, void* output, uint32_t blocks); -/** - * @brief C ABI wrapper for decompressing multiple single-precision blocks. - * - * @param bit_width bit width per packed value (1 to 24). - * @param input pointer to the first packed block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @param blocks number of consecutive blocks to process. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); +PERNIX_API pernix_status pernix_decompress_block_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + double scale, void* output, bool sign_values); -/** - * @brief C ABI wrapper for decompressing multiple double-precision blocks. - * - * @param bit_width bit width per packed value (1 to 24). - * @param input pointer to the first packed block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @param blocks number of consecutive blocks to process. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); +PERNIX_API pernix_status pernix_decompress_blocks_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + double scale, void* output, uint32_t blocks, bool sign_values); -#ifdef __cplusplus +#if defined(__cplusplus) } -} // namespace pernix #endif -#endif // PERNIX_H +#endif //PERNIX_H diff --git a/include/pernix/pernix.hpp b/include/pernix/pernix.hpp new file mode 100644 index 0000000..67301bb --- /dev/null +++ b/include/pernix/pernix.hpp @@ -0,0 +1,153 @@ +#ifndef PERNIX_HPP +#define PERNIX_HPP +#include + +namespace pernix { +enum class Backend { + Auto = PERNIX_BACKEND_AUTO, + Fallback = PERNIX_BACKEND_FALLBACK, + X86Avx2 = PERNIX_BACKEND_X86_AVX2, + X86Bmi2 = PERNIX_BACKEND_X86_BMI2, + X86Avx512Vbmi = PERNIX_BACKEND_X86_AVX512_VBMI, + Arm64Neon = PERNIX_BACKEND_ARM64_NEON, + Arm64Sve = PERNIX_BACKEND_ARM64_SVE +}; + +__always_inline int compress_block(Backend backend, const uint8_t bit_width, const uint32_t block_size, + const std::span input, const float scale, std::span output) { + return pernix_compress_block_f32(static_cast(backend), bit_width, block_size, input.data(), scale, output.data()); +} + +__always_inline int compress_block(Backend backend, const uint8_t bit_width, const uint32_t block_size, + const std::span input, const double scale, std::span output) { + return pernix_compress_block_f64(static_cast(backend), bit_width, block_size, input.data(), scale, output.data()); +} + +__always_inline int decompress_block(Backend backend, const uint8_t bit_width, const uint32_t block_size, + const std::span input, const float scale, std::span output, + const bool sign_values = true) { + return pernix_decompress_block_f32(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), + sign_values); +} + +__always_inline int decompress_block(Backend backend, const uint8_t bit_width, const uint32_t block_size, + const std::span input, const double scale, std::span output, + const bool sign_values = true) { + return pernix_decompress_block_f64(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), + sign_values); +} + +__always_inline int compress_blocks(Backend backend, const uint8_t bit_width, const uint32_t block_size, + const std::span input, const float scale, std::span output, + const uint32_t blocks) { + return pernix_compress_blocks_f32(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), + blocks); +} + +__always_inline int compress_blocks(Backend backend, const uint8_t bit_width, const uint32_t block_size, + const std::span input, const double scale, std::span output, + const uint32_t blocks) { + return pernix_compress_blocks_f64(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), + blocks); +} + +__always_inline int decompress_blocks(Backend backend, const uint8_t bit_width, const uint32_t block_size, + const std::span input, const float scale, std::span output, + const uint32_t blocks, const bool sign_values = true) { + return pernix_decompress_blocks_f32(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), + blocks, sign_values); +} + +__always_inline int decompress_blocks(Backend backend, const uint8_t bit_width, const uint32_t block_size, + const std::span input, const double scale, std::span output, + const uint32_t blocks, const bool sign_values = true) { + return pernix_decompress_blocks_f64(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), + blocks, sign_values); +} + +// convenience overloads without backend (defaults to Auto) +__always_inline int compress_block(const uint8_t bit_width, const uint32_t block_size, const std::span input, + const float scale, const std::span output) { + return compress_block(Backend::Auto, bit_width, block_size, input, scale, output); +} + +__always_inline int compress_block(const uint8_t bit_width, const uint32_t block_size, const std::span input, + const double scale, const std::span output) { + return compress_block(Backend::Auto, bit_width, block_size, input, scale, output); +} + +__always_inline int decompress_block(const uint8_t bit_width, const uint32_t block_size, const std::span input, + const float scale, const std::span output, const bool sign_values = true) { + return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); +} + +__always_inline int decompress_block(const uint8_t bit_width, const uint32_t block_size, const std::span input, + const double scale, const std::span output, const bool sign_values = true) { + return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); +} + +__always_inline int compress_blocks(const uint8_t bit_width, const uint32_t block_size, const std::span input, + const float scale, const std::span output, const uint32_t blocks) { + return compress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks); +} + +__always_inline int compress_blocks(const uint8_t bit_width, const uint32_t block_size, const std::span input, + const double scale, const std::span output, const uint32_t blocks) { + return compress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks); +} + +__always_inline int decompress_blocks(const uint8_t bit_width, const uint32_t block_size, const std::span input, + const float scale, const std::span output, const uint32_t blocks, + const bool sign_values = true) { + return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); +} + +__always_inline int decompress_blocks(const uint8_t bit_width, const uint32_t block_size, const std::span input, + const double scale, const std::span output, const uint32_t blocks, + const bool sign_values = true) { + return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); +} + +// convenience overloads without backend and without block_size (defaults to 64) +__always_inline int compress_block(const uint8_t bit_width, const std::span input, const float scale, + const std::span output) { + return compress_block(Backend::Auto, bit_width, 64, input, scale, output); +} + +__always_inline int compress_block(const uint8_t bit_width, const std::span input, const double scale, + const std::span output) { + return compress_block(Backend::Auto, bit_width, 64, input, scale, output); +} + +__always_inline int decompress_block(const uint8_t bit_width, const std::span input, const float scale, + const std::span output, const bool sign_values = true) { + return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); +} + +__always_inline int decompress_block(const uint8_t bit_width, const std::span input, const double scale, + const std::span output, const bool sign_values = true) { + return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); +} + +__always_inline int compress_blocks(const uint8_t bit_width, const std::span input, const float scale, + const std::span output, const uint32_t blocks) { + return compress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks); +} + +__always_inline int compress_blocks(const uint8_t bit_width, const std::span input, const double scale, + const std::span output, const uint32_t blocks) { + return compress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks); +} + +__always_inline int decompress_blocks(const uint8_t bit_width, const std::span input, const float scale, + const std::span output, const uint32_t blocks, const bool sign_values = true) { + return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); +} + +__always_inline int decompress_blocks(const uint8_t bit_width, const std::span input, const double scale, + const std::span output, const uint32_t blocks, const bool sign_values = true) { + return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); +} +} + +#endif //PERNIX_HPP diff --git a/include/pernix/x86/avx512vbmi/compat.h b/include/pernix/x86/avx512vbmi/compat.h deleted file mode 100644 index bbc7ad4..0000000 --- a/include/pernix/x86/avx512vbmi/compat.h +++ /dev/null @@ -1,387 +0,0 @@ -#ifndef PERNIX_AVX512_COMPAT_H -#define PERNIX_AVX512_COMPAT_H - -#include -#include -#include -#include - -namespace pernix::internal { -static __always_inline __mmask8 element_mask8(const uint32_t e) { - return static_cast<__mmask8>(e >= 8 ? 0xFFu : ((1u << e) - 1u)); -} - -static __always_inline __mmask16 element_mask16(const uint32_t e) { - return static_cast<__mmask16>(e >= 16 ? 0xFFFFu : ((1u << e) - 1u)); -} - -static __always_inline __mmask32 element_mask32(const uint32_t e) { - return e >= 32 ? 0xFFFFFFFFu : (1u << e) - 1u; -} - -static __always_inline __mmask64 element_mask64(const uint32_t e) { - return e >= 64 ? 0xFFFFFFFFFFFFFFFFull : (1ull << e) - 1ull; -} - -static __always_inline __m512i mm512_loadu_elements_epi64(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int64_t)); - return a; -#else - return _mm512_maskz_loadu_epi64(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m256i mm256_loadu_elements_epi64(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int64_t)); - return a; -#else - return _mm256_maskz_loadu_epi64(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m128i mm_loadu_elements_epi64(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int64_t)); - return a; -#else - return _mm_maskz_loadu_epi64(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m512i mm512_loadu_elements_epi32(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int32_t)); - return a; -#else - return _mm512_maskz_loadu_epi32(element_mask16(e), mem_addr); -#endif -} - -static __always_inline __m256i mm256_loadu_elements_epi32(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int32_t)); - return a; -#else - return _mm256_maskz_loadu_epi32(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m128i mm_loadu_elements_epi32(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int32_t)); - return a; -#else - return _mm_maskz_loadu_epi32(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m512i mm512_loadu_elements_epi16(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int16_t)); - return a; -#else - return _mm512_maskz_loadu_epi16(element_mask32(e), mem_addr); -#endif -} - -static __always_inline __m256i mm256_loadu_elements_epi16(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int16_t)); - return a; -#else - return _mm256_maskz_loadu_epi16(element_mask16(e), mem_addr); -#endif -} - -static __always_inline __m128i mm_loadu_elements_epi16(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int16_t)); - return a; -#else - return _mm_maskz_loadu_epi16(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m512i mm512_loadu_elements_epi8(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int8_t)); - return a; -#else - return _mm512_maskz_loadu_epi8(element_mask64(e), mem_addr); -#endif -} - -static __always_inline __m256i mm256_loadu_elements_epi8(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int8_t)); - return a; -#else - return _mm256_maskz_loadu_epi8(element_mask32(e), mem_addr); -#endif -} - -static __always_inline __m128i mm_loadu_elements_epi8(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int8_t)); - return a; -#else - return _mm_maskz_loadu_epi8(element_mask16(e), mem_addr); -#endif -} - -static __always_inline void mm512_storeu_elements_epi64(void* mem_addr, const uint32_t e, const __m512i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; - _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); -#else - _mm512_mask_storeu_epi64(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm256_storeu_elements_epi64(void* mem_addr, const uint32_t e, const __m256i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; - _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); -#else - _mm256_mask_storeu_epi64(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm_storeu_elements_epi64(void* mem_addr, const uint32_t e, const __m128i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; - _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); -#else - _mm_mask_storeu_epi64(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm512_storeu_elements_epi32(void* mem_addr, const uint32_t e, const __m512i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; - _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); -#else - _mm512_mask_storeu_epi32(mem_addr, element_mask16(e), a); -#endif -} - -static __always_inline void mm256_storeu_elements_epi32(void* mem_addr, const uint32_t e, const __m256i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; - _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); -#else - _mm256_mask_storeu_epi32(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm_storeu_elements_epi32(void* mem_addr, const uint32_t e, const __m128i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; - _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); -#else - _mm_mask_storeu_epi32(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm512_storeu_elements_epi16(void* mem_addr, const uint32_t e, const __m512i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; - _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); -#else - _mm512_mask_storeu_epi16(mem_addr, element_mask32(e), a); -#endif -} - -static __always_inline void mm256_storeu_elements_epi16(void* mem_addr, const uint32_t e, const __m256i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; - _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); -#else - _mm256_mask_storeu_epi16(mem_addr, element_mask16(e), a); -#endif -} - -static __always_inline void mm_storeu_elements_epi16(void* mem_addr, const uint32_t e, const __m128i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; - _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); -#else - _mm_mask_storeu_epi16(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm512_storeu_elements_epi8(void* mem_addr, const uint32_t e, const __m512i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; - _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); -#else - _mm512_mask_storeu_epi8(mem_addr, element_mask64(e), a); -#endif -} - -static __always_inline void mm256_storeu_elements_epi8(void* mem_addr, const uint32_t e, const __m256i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; - _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); -#else - _mm256_mask_storeu_epi8(mem_addr, element_mask32(e), a); -#endif -} - -static __always_inline void mm_storeu_elements_epi8(void* mem_addr, const uint32_t e, const __m128i a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; - _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); -#else - _mm_mask_storeu_epi8(mem_addr, element_mask16(e), a); -#endif -} - -static __always_inline __m512 mm512_loadu_elements_ps(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512 a = _mm512_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(float_t)); - return a; -#else - return _mm512_maskz_loadu_ps(element_mask16(e), mem_addr); -#endif -} - -static __always_inline __m256 mm256_loadu_elements_ps(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256 a = _mm256_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(float_t)); - return a; -#else - return _mm256_maskz_loadu_ps(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m128 mm_loadu_elements_ps(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128 a = _mm_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(float_t)); - return a; -#else - return _mm_maskz_loadu_ps(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m512d mm512_loadu_elements_pd(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512d a = _mm512_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(double_t)); - return a; -#else - return _mm512_maskz_loadu_pd(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m256d mm256_loadu_elements_pd(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256d a = _mm256_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(double_t)); - return a; -#else - return _mm256_maskz_loadu_pd(element_mask8(e), mem_addr); -#endif -} - -static __always_inline __m128d mm_loadu_elements_pd(const uint32_t e, const void* mem_addr) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128d a = _mm_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(double_t)); - return a; -#else - return _mm_maskz_loadu_pd(element_mask8(e), mem_addr); -#endif -} - -static __always_inline void mm512_storeu_elements_ps(void* mem_addr, const uint32_t e, const __m512 a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) float_t values[16]; - _mm512_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(float_t)); -#else - _mm512_mask_storeu_ps(mem_addr, element_mask16(e), a); -#endif -} - -static __always_inline void mm256_storeu_elements_ps(void* mem_addr, const uint32_t e, const __m256 a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) float_t values[8]; - _mm256_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(float_t)); -#else - _mm256_mask_storeu_ps(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm_storeu_elements_ps(void* mem_addr, const uint32_t e, const __m128 a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) float_t values[4]; - _mm_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(float_t)); -#else - _mm_mask_storeu_ps(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm512_storeu_elements_pd(void* mem_addr, const uint32_t e, const __m512d a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) double_t values[8]; - _mm512_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(double_t)); -#else - _mm512_mask_storeu_pd(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm256_storeu_elements_pd(void* mem_addr, const uint32_t e, const __m256d a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) double_t values[4]; - _mm256_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(double_t)); -#else - _mm256_mask_storeu_pd(mem_addr, element_mask8(e), a); -#endif -} - -static __always_inline void mm_storeu_elements_pd(void* mem_addr, const uint32_t e, const __m128d a) { -#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) double_t values[2]; - _mm_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(double_t)); -#else - _mm_mask_storeu_pd(mem_addr, element_mask8(e), a); -#endif -} -} - -#endif //PERNIX_AVX512_COMPAT_H diff --git a/include/pernix/x86/avx512vbmi/packing.h b/include/pernix/x86/avx512vbmi/packing.h deleted file mode 100644 index ba3b132..0000000 --- a/include/pernix/x86/avx512vbmi/packing.h +++ /dev/null @@ -1,327 +0,0 @@ -#ifndef PERNIX_AVX512VBMI_PACKING_H -#define PERNIX_AVX512VBMI_PACKING_H - -#include -#include - -namespace pernix::internal { -namespace m128 { -/** - * @brief Pack 8 16-bit values for bit widths 9 through 16 using VBMI. - */ -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = pack_tables_avx512_16; - const __m128i maskv = _mm_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i permuted1 = _mm_permutexvar_epi16(tables::get_permute1(), masked); - const __m128i permuted2 = _mm_permutexvar_epi16(tables::get_permute2(), masked); - - const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm_or_si128(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m128i permuted1 = _mm_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m128i permuted2 = _mm_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m128i permuted3 = _mm_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); - - const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); - } - } -} - -/** - * @brief Pack 16 8-bit values for bit widths 1 through 8 using VBMI. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { - if constexpr (BIT_WIDTH == 8) { - return input; - } else { - const __m128i maskv = _mm_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm_set1_epi16(static_cast(_mm_cmpgt_epi8_mask(masked, _mm_setzero_si128()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m128i shifted = _mm_srli_epi16(masked, 6); - const __m128i combined = _mm_or_si128(masked, shifted); - - const __m128i shifted2 = _mm_srli_epi32(combined, 12); - const __m128i combined2 = _mm_or_si128(shifted2, combined); - - return _mm_cvtepi32_epi8(combined2); - } else if constexpr (BIT_WIDTH == 3) { - const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); - - const __m128i pair6 = _mm_or_si128(even, _mm_srli_epi16(odd, 5)); - const __m128i packed12 = _mm_or_si128(pair6, _mm_srli_epi32(pair6, 10)); - - return m128::mm_pack_epi16_avx512vbmi_9to16<12>(_mm_cvtepi32_epi16(packed12)); - } else if constexpr (BIT_WIDTH == 4) { - const __m128i shifted = _mm_srli_epi16(masked, 4); - const __m128i combined = _mm_or_si128(masked, shifted); - - return _mm_cvtepi16_epi8(combined); - } else { - const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); - - const __m128i shifted = _mm_or_si128(even, _mm_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } - } -} - -/** - * @brief Pack 4 32-bit values for bit widths 17 through 24 using VBMI. - */ -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i& input) { - using tables = pack_tables_avx512_24; - - const __m128i maskv = _mm_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - - const __m128 permuted1 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute1()); - const __m128 permuted2 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute2()); - const __m128 permuted3 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute3()); - - const __m128i shifted1 = _mm_sllv_epi32(_mm_castps_si128(permuted1), tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi32(_mm_castps_si128(permuted2), tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi32(_mm_castps_si128(permuted3), tables::get_shift3()); - - return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); -} -} // namespace m128 - -namespace m256 { -/** - * @brief Pack 16 16-bit values for bit widths 9 through 16 using VBMI. - */ -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = pack_tables_avx512_16; - const __m256i maskv = _mm256_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m256i permuted1 = _mm256_permutexvar_epi16(tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_permutexvar_epi16(tables::get_permute2(), masked); - - const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm256_or_si256(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); - - const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); - } - } -} - -/** - * @brief Pack 32 8-bit values for bit widths 1 through 8 using VBMI. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { - if constexpr (BIT_WIDTH == 8) { - return input; - } else { - const __m256i maskv = _mm256_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm256_set1_epi32(static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m256i shifted = _mm256_srli_epi16(masked, 6); - const __m256i combined = _mm256_or_si256(masked, shifted); - - const __m256i shifted2 = _mm256_srli_epi32(combined, 12); - const __m256i combined2 = _mm256_or_si256(shifted2, combined); - - return _mm256_castsi128_si256(_mm256_cvtepi32_epi8(combined2)); - } else if constexpr (BIT_WIDTH == 3) { - const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); - - const __m256i pair6 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 5)); - const __m256i packed12 = _mm256_or_si256(pair6, _mm256_srli_epi32(pair6, 10)); - - return m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm256_castsi128_si256(_mm256_cvtepi32_epi16(packed12))); - } else if constexpr (BIT_WIDTH == 4) { - const __m256i shifted = _mm256_srli_epi16(masked, 4); - const __m256i combined = _mm256_or_si256(masked, shifted); - - return _mm256_castsi128_si256(_mm256_cvtepi16_epi8(combined)); - } else { - const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); - - const __m256i shifted = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm256_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } - } -} - -/** - * @brief Pack 8 32-bit values for bit widths 17 through 24 using VBMI. - */ -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i& input) { - using tables = pack_tables_avx512_24; - - const __m256i maskv = _mm256_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - const __m256i permuted1 = _mm256_permutexvar_epi32(tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_permutexvar_epi32(tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_permutexvar_epi32(tables::get_permute3(), masked); - - const __m256i shifted1 = _mm256_sllv_epi32(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi32(permuted2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi32(permuted3, tables::get_shift3()); - - return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); -} -} // namespace m256 - -namespace m512 { -/** - * @brief Pack 32 16-bit values for bit widths 9 through 16 using VBMI. - */ -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = pack_tables_avx512_16; - const __m512i maskv = _mm512_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m512i permuted1 = _mm512_permutexvar_epi16(tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_permutexvar_epi16(tables::get_permute2(), masked); - - const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm512_or_si512(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); - - const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); - const __m512i shifted3 = _mm512_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); - } - } -} - -/** - * @brief Pack 64 8-bit values for bit widths 1 through 8 using VBMI. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { - if constexpr (BIT_WIDTH == 8) { - return input; - } else { - const __m512i maskv = _mm512_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm512_set1_epi64(static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m512i shifted = _mm512_srli_epi16(masked, 6); - const __m512i combined = _mm512_or_si512(masked, shifted); - - const __m512i shifted2 = _mm512_srli_epi32(combined, 12); - const __m512i combined2 = _mm512_or_si512(shifted2, combined); - - return _mm512_castsi128_si512(_mm512_cvtepi32_epi8(combined2)); - } else if constexpr (BIT_WIDTH == 3) { - const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); - - const __m512i pair6 = _mm512_or_si512(even, _mm512_srli_epi16(odd, 5)); - const __m512i packed12 = _mm512_or_si512(pair6, _mm512_srli_epi32(pair6, 10)); - - return _mm512_castsi256_si512(m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm512_cvtepi32_epi16(packed12))); - } else if constexpr (BIT_WIDTH == 4) { - const __m512i shifted = _mm512_srli_epi16(masked, 4); - const __m512i combined = _mm512_or_si512(masked, shifted); - - return _mm512_castsi256_si512(_mm512_cvtepi16_epi8(combined)); - } else { - const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); - - const __m512i shifted = _mm512_or_si512(even, _mm512_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm512_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } - } -} - -/** - * @brief Pack 16 32-bit values for bit widths 17 through 24 using VBMI. - */ -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i& input) { - using tables = pack_tables_avx512_24; - - const __m512i maskv = _mm512_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - const __m512i permuted1 = _mm512_permutexvar_epi32(tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_permutexvar_epi32(tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_permutexvar_epi32(tables::get_permute3(), masked); - - const __m512i shifted1 = _mm512_sllv_epi32(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi32(permuted2, tables::get_shift2()); - const __m512i shifted3 = _mm512_srlv_epi32(permuted3, tables::get_shift3()); - - return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); -} -} // namespace m512 -} // namespace pernix::internal - -#endif // PERNIX_AVX512VBMI_PACKING_H diff --git a/include/pernix/x86/avx512vbmi/unpacking.h b/include/pernix/x86/avx512vbmi/unpacking.h deleted file mode 100644 index 1cfc29c..0000000 --- a/include/pernix/x86/avx512vbmi/unpacking.h +++ /dev/null @@ -1,500 +0,0 @@ -#ifndef PERNIX_AVX512VBMI_UNPACKING_H -#define PERNIX_AVX512VBMI_UNPACKING_H - -#include -#include - -namespace pernix::internal { -namespace m128 { -constexpr __mmask16 kAlternateByteMask16 = 0xAAAAULL; - -__always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0x00ff); - const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); - const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); - return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); -} - -__always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0xff00); - const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); - const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); - return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); -} - -__always_inline static __m128i _mm_slli_epi8(const __m128i a, const int8_t imm8) { - return _mm_sllv_epi8(a, _mm_set1_epi8(imm8)); -} - -__always_inline static __m128i _mm_srli_epi8(const __m128i a, const int imm8) { - const __m128i lo_mask = _mm_set1_epi16(0x00ff); - const __m128i hi_mask = _mm_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); - - const __m128i lo = _mm_srl_epi16(_mm_and_si128(a, lo_mask), shift); - const __m128i hi = _mm_and_si128(_mm_srl_epi16(a, shift), hi_mask); - - return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); -} - -__always_inline static __m128i _mm_srai_epi8(const __m128i a, const int8_t imm8) { - const __m128i lo_mask = _mm_set1_epi16(0x00ff); - const __m128i hi_mask = _mm_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); - - const __m128i hi = _mm_and_si128(_mm_sra_epi16(a, shift), hi_mask); - - const __m128i lo_as_hi = _mm_slli_epi16(_mm_and_si128(a, lo_mask), 8); - const __m128i lo = _mm_and_si128(_mm_srli_epi16(_mm_sra_epi16(lo_as_hi, shift), 8), lo_mask); - - return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i& input) { - if constexpr (BIT_WIDTH == 8) { - return input; - } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask16>(_mm_cvtsi128_si64(input)); - const __m128i source = _mm_movm_epi8(value); - const __m128i unpacked = _mm_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m128i values_shift0 = input; - __m128i values_shift2 = _mm_srli_epi16(values_shift0, 2); - const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); - const __m128i values_shift6 = _mm_srli_epi16(values_shift0, 6); - - __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - - interleave_tmp = _mm_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm_unpacklo_epi64(interleave_tmp, values_shift2); - - interleave_tmp = _mm_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); - - values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0303)); - - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m128i values_shift0 = input; - const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); - - const __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); - - values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0F0F)); - - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m128i permuted1 = _mm_permutexvar_epi8(tables::get_permute1(), input); - const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); - - const __m128i shifted1 = _mm_srlv_epi8(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask16 spill_mask = _mm_cmpneq_epi8_mask(tables::get_shift2(), _mm_setzero_si128()); - __m128i combined = _mm_or_si128(shifted1, _mm_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr uint32_t shift = 8 - BIT_WIDTH; - combined = _mm_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm_srai_epi8(combined, shift); - } else { - combined = _mm_srli_epi8(combined, shift); - } - - return combined; - } - } -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i& input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = unpack_tables_avx512_16; - - const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute1(), input); - - __m128i shifted = _mm_srlv_epi16(permuted, tables::get_shift1()); - - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); - const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); - shifted = _mm_or_si128(shifted, shifted2); - } - - constexpr uint32_t shift = 16 - BIT_WIDTH; - shifted = _mm_slli_epi16(shifted, shift); - if (SIGN_VALUES) { - shifted = _mm_srai_epi16(shifted, shift); - } else { - shifted = _mm_srli_epi16(shifted, shift); - } - - return shifted; - } -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i& input) { - using tables = unpack_tables_avx512_24; - - const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute(), input); - - constexpr uint32_t shift = 32 - BIT_WIDTH; - __m128i shifted = _mm_sllv_epi32(permuted, tables::get_shift()); - if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { - shifted = _mm_srai_epi32(shifted, shift); - } else { - shifted = _mm_srli_epi32(shifted, shift); - } - - return shifted; -} -} // namespace m128 - -namespace m256 { -constexpr __mmask32 kAlternateByteMask32 = 0xAAAAAAAAULL; - -__always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0x00ff); - const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); - const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); - return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); -} - -__always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0xff00); - const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); - const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); - return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); -} - -__always_inline static __m256i _mm256_slli_epi8(const __m256i a, const int8_t imm8) { - return _mm256_sllv_epi8(a, _mm256_set1_epi8(imm8)); -} - -__always_inline static __m256i _mm256_srli_epi8(const __m256i a, const int8_t imm8) { - const __m256i lo_mask = _mm256_set1_epi16(0x00ff); - const __m256i hi_mask = _mm256_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); - - const __m256i lo = _mm256_srl_epi16(_mm256_and_si256(a, lo_mask), shift); - const __m256i hi = _mm256_and_si256(_mm256_srl_epi16(a, shift), hi_mask); - - return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); -} - -__always_inline static __m256i _mm256_srai_epi8(const __m256i a, const int8_t imm8) { - const __m256i lo_mask = _mm256_set1_epi16(0x00ff); - const __m256i hi_mask = _mm256_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); - - const __m256i hi = _mm256_and_si256(_mm256_sra_epi16(a, shift), hi_mask); - - const __m256i lo_as_hi = _mm256_slli_epi16(_mm256_and_si256(a, lo_mask), 8); - const __m256i lo = _mm256_and_si256(_mm256_srli_epi16(_mm256_sra_epi16(lo_as_hi, shift), 8), lo_mask); - - return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i& input) { - if constexpr (BIT_WIDTH == 8) { - return input; - } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask32>(_mm_cvtsi128_si64(_mm256_castsi256_si128(input))); - const __m256i source = _mm256_movm_epi8(value); - const __m256i unpacked = _mm256_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m256i values_shift0 = input; - __m256i values_shift2 = _mm256_srli_epi16(values_shift0, 2); - const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); - const __m256i values_shift6 = _mm256_srli_epi16(values_shift0, 6); - - __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); - - interleave_tmp = _mm256_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm256_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm256_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); - - interleave_tmp = _mm256_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm256_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); - values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); - - values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0303)); - - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m256i values_shift0 = input; - const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); - - __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); - values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); - - values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0F0F)); - - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m256i permuted1 = _mm256_permutexvar_epi8(tables::get_permute1(), input); - const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); - - const __m256i shifted1 = _mm256_srlv_epi8(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask32 spill_mask = _mm256_cmpneq_epi8_mask(tables::get_shift2(), _mm256_setzero_si256()); - __m256i combined = _mm256_or_si256(shifted1, _mm256_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr uint32_t shift = 8 - BIT_WIDTH; - combined = _mm256_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm256_srai_epi8(combined, shift); - } else { - combined = _mm256_srli_epi8(combined, shift); - } - - return combined; - } - } -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i& input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = unpack_tables_avx512_16; - - const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute1(), input); - - __m256i shifted = _mm256_srlv_epi16(permuted, tables::get_shift1()); - - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); - const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); - shifted = _mm256_or_si256(shifted, shifted2); - } - - constexpr uint32_t shift = 16 - BIT_WIDTH; - shifted = _mm256_slli_epi16(shifted, shift); - if (SIGN_VALUES) { - shifted = _mm256_srai_epi16(shifted, shift); - } else { - shifted = _mm256_srli_epi16(shifted, shift); - } - - return shifted; - } -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i& input) { - using tables = unpack_tables_avx512_24; - - const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute(), input); - - constexpr uint32_t shift = 32 - BIT_WIDTH; - __m256i shifted = _mm256_sllv_epi32(permuted, tables::get_shift()); - if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { - shifted = _mm256_srai_epi32(shifted, shift); - } else { - shifted = _mm256_srli_epi32(shifted, shift); - } - - return shifted; -} -} // namespace m256 - -namespace m512 { -constexpr __mmask64 kAlternateByteMask64 = 0xAAAAAAAAAAAAAAAAULL; - -__always_inline static __m512i _mm512_srlv_epi8(const __m512i a, const __m512i count) { - const __m512i mask = _mm512_set1_epi16(0x00ff); - const __m512i low_half = _mm512_srlv_epi16(_mm512_and_si512(mask, a), _mm512_and_si512(mask, count)); - const __m512i high_half = _mm512_srlv_epi16(a, _mm512_srli_epi16(count, 8)); - return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); -} - -__always_inline static __m512i _mm512_sllv_epi8(const __m512i a, const __m512i count) { - const __m512i mask = _mm512_set1_epi16(0xff00); - const __m512i low_half = _mm512_sllv_epi16(a, _mm512_andnot_si512(mask, count)); - const __m512i high_half = _mm512_sllv_epi16(_mm512_and_si512(mask, a), _mm512_srli_epi16(count, 8)); - return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); -} - -__always_inline static __m512i _mm512_slli_epi8(const __m512i a, const int8_t imm8) { - return _mm512_sllv_epi8(a, _mm512_set1_epi8(imm8)); -} - -__always_inline static __m512i _mm512_srli_epi8(const __m512i a, const int8_t imm8) { - const __m512i lo_mask = _mm512_set1_epi16(0x00ff); - const __m512i hi_mask = _mm512_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); - - const __m512i lo = _mm512_srl_epi16(_mm512_and_si512(a, lo_mask), shift); - const __m512i hi = _mm512_and_si512(_mm512_srl_epi16(a, shift), hi_mask); - - return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); -} - -__always_inline static __m512i _mm512_srai_epi8(const __m512i a, const int8_t imm8) { - const __m512i lo_mask = _mm512_set1_epi16(0x00ff); - const __m512i hi_mask = _mm512_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); - - const __m512i hi = _mm512_and_si512(_mm512_sra_epi16(a, shift), hi_mask); - - const __m512i lo_as_hi = _mm512_slli_epi16(_mm512_and_si512(a, lo_mask), 8); - const __m512i lo = _mm512_and_si512(_mm512_srli_epi16(_mm512_sra_epi16(lo_as_hi, shift), 8), lo_mask); - - return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i& input) { - if constexpr (BIT_WIDTH == 8) { - return input; - } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask64>(_mm_cvtsi128_si64(_mm512_castsi512_si128(input))); - const __m512i source = _mm512_movm_epi8(value); - const __m512i unpacked = _mm512_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m512i values_shift0 = input; - __m512i values_shift2 = _mm512_srli_epi16(values_shift0, 2); - const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); - const __m512i values_shift6 = _mm512_srli_epi16(values_shift0, 6); - - __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); - - interleave_tmp = _mm512_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm512_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm512_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); - - interleave_tmp = _mm512_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm512_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x88); - values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); - - values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0303)); - - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m512i values_shift0 = input; - const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); - - __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x44); - values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); - - values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0F0F)); - - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m512i permuted1 = _mm512_permutexvar_epi8(tables::get_permute1(), input); - const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); - - const __m512i shifted1 = _mm512_srlv_epi8(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask64 spill_mask = _mm512_cmpneq_epi8_mask(tables::get_shift2(), _mm512_setzero_si512()); - __m512i combined = _mm512_or_si512(shifted1, _mm512_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr uint32_t shift = 8 - BIT_WIDTH; - combined = _mm512_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm512_srai_epi8(combined, shift); - } else { - combined = _mm512_srli_epi8(combined, shift); - } - - return combined; - } - } -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i& input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = unpack_tables_avx512_16; - - const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute1(), input); - __m512i shifted = _mm512_srlv_epi16(permuted, tables::get_shift1()); - - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); - const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); - shifted = _mm512_or_si512(shifted, shifted2); - } - - constexpr uint32_t shift = 16 - BIT_WIDTH; - shifted = _mm512_slli_epi16(shifted, shift); - if (SIGN_VALUES) { - shifted = _mm512_srai_epi16(shifted, shift); - } else { - shifted = _mm512_srli_epi16(shifted, shift); - } - - return shifted; - } -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i& input) { - using tables = unpack_tables_avx512_24; - - const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute(), input); - __m512i shifted = _mm512_sllv_epi32(permuted, tables::get_shift()); - - constexpr uint32_t shift = 32 - BIT_WIDTH; - if constexpr (SIGN_VALUES) { - shifted = _mm512_srai_epi32(shifted, shift); - } else { - shifted = _mm512_srli_epi32(shifted, shift); - } - - return shifted; -} -} // namespace m512 -} // namespace pernix::internal - -#endif // PERNIX_AVX512VBMI_UNPACKING_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 57d2f0a..aab5c67 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,12 +1,204 @@ include(GNUInstallDirs) include(CMakePackageConfigHelpers) +add_library(pernix SHARED) +target_sources(pernix + PRIVATE + pernix.cpp + + fallback/fallback_compression.cpp + fallback/fallback_decompression.cpp + + dispatch/select.cpp +) + +target_compile_features(pernix PUBLIC cxx_std_20) + +set_target_properties(pernix PROPERTIES + OUTPUT_NAME "pernix" + VERSION ${NORMALIZED_VERSION} + LINKER_LANGUAGE CXX +) + +target_include_directories(pernix + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/internal +) + +target_compile_definitions(pernix + PRIVATE + PERNIX_BUILD_LIB=1 +) + +if (PERNIX_USE_SIMDE) + # target_link_libraries(pernix PUBLIC simde::simde) + target_compile_definitions(pernix PUBLIC PERNIX_USE_SIMDE=1) +endif () + +if (PERNIX_TARGET_IS_X86) + target_sources(pernix + PRIVATE + dispatch/cpu_features_x86.cpp + ) + + if (MSVC) + set_source_files_properties( + dispatch/cpu_features_x86.cpp + PROPERTIES + COMPILE_OPTIONS "/arch:AVX512" + ) + else () + set_source_files_properties( + dispatch/cpu_features_x86.cpp + PROPERTIES + COMPILE_OPTIONS "-mavx;-mbmi;-mbmi2;-mavx2;-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512cd;-mavx512vbmi" + ) + endif () +endif () + +if (PERNIX_TARGET_IS_ARM64) + target_sources(pernix + PRIVATE + dispatch/cpu_features_arm.cpp + ) + + set_target_properties(pernix PROPERTIES + COMPILE_OPTIONS "-march=armv8-a+simd+sve2" + ) +endif () + +if (PERNIX_ENABLE_X86_BMI2 AND PERNIX_TARGET_IS_X86) + target_sources(pernix + PRIVATE + x86/bmi2/bmi2_compression.cpp + x86/bmi2/bmi2_decompression.cpp + ) + + target_compile_definitions(pernix + PRIVATE + PERNIX_BUILD_X86_BMI2=1 + ) + + if (MSVC) + # BMI2 is not exposed with MSVC + else () + set_source_files_properties( + x86/bmi2/bmi2_compression.cpp + x86/bmi2/bmi2_decompression.cpp + PROPERTIES + COMPILE_OPTIONS "-mbmi;-mbmi2;-mavx2" + ) + endif () + +endif () + +if (PERNIX_ENABLE_X86_AVX2 AND PERNIX_TARGET_IS_X86) + target_sources(pernix + PRIVATE + x86/avx2/avx2_compression.cpp + x86/avx2/avx2_decompression.cpp + ) + + target_compile_definitions(pernix + PRIVATE + PERNIX_BUILD_X86_AVX2=1 + ) + + if (MSVC) + set_source_files_properties( + x86/avx2/avx2_compression.cpp + x86/avx2/avx2_decompression.cpp + PROPERTIES + COMPILE_OPTIONS "/arch:AVX2" + ) + else () + set_source_files_properties( + x86/avx2/avx2_compression.cpp + x86/avx2/avx2_decompression.cpp + PROPERTIES + COMPILE_OPTIONS "-mavx2" + ) + endif () +endif () + +if (PERNIX_ENABLE_X86_AVX512VBMI AND PERNIX_TARGET_IS_X86) + target_sources(pernix + PRIVATE + x86/avx512vbmi/avx512vbmi_compression.cpp + x86/avx512vbmi/avx512vbmi_decompression.cpp + ) + + target_compile_definitions(pernix + PRIVATE + PERNIX_BUILD_X86_AVX512_VBMI=1 + ) + + if (MSVC) + set_source_files_properties( + x86/avx512vbmi/avx512vbmi_compression.cpp + x86/avx512vbmi/avx512vbmi_decompression.cpp + PROPERTIES + COMPILE_OPTIONS "/arch:AVX512" + ) + else () + set_source_files_properties( + x86/avx512vbmi/avx512vbmi_compression.cpp + x86/avx512vbmi/avx512vbmi_decompression.cpp + PROPERTIES + COMPILE_OPTIONS "-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512cd;-mavx512vbmi" + ) + endif () +endif () + +if (PERNIX_ENABLE_ARM64_NEON AND PERNIX_TARGET_IS_ARM64) + target_sources(pernix + PRIVATE + arm64/neon/compression.cpp + arm64/neon/decompression.cpp + ) + + target_compile_definitions(pernix + PRIVATE + PERNIX_BUILD_ARM64_NEON=1 + ) + + set_source_files_properties( + arm64/neon/compression.cpp + arm64/neon/decompression.cpp + PROPERTIES + COMPILE_OPTIONS "-march=armv8-a+simd" + ) +endif () + +if (PERNIX_ENABLE_ARM64_SVE2 AND PERNIX_TARGET_IS_ARM64) + target_sources(pernix + PRIVATE + arm64/sve2/compression.cpp + arm64/sve2/decompression.cpp + ) + + target_compile_definitions(pernix + PRIVATE + PERNIX_BUILD_ARM64_SVE2=1 + ) + + set_source_files_properties( + arm64/sve2/compression.cpp + arm64/sve2/decompression.cpp + PROPERTIES + COMPILE_OPTIONS "-march=armv8.2-a+simd+sve2" + ) +endif () + +#[===[ file(GLOB_RECURSE PERNIX_COMMON_SOURCES - CONFIGURE_DEPENDS ./fallback/*.cpp - ./pernix.cpp - ${PROJECT_SOURCE_DIR}/include/pernix/*.h + pernix.cpp + dispatch/select.cpp ) set(PERNIX_SOURCES ${PERNIX_COMMON_SOURCES}) @@ -16,7 +208,6 @@ if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "X86") PERNIX_X86_SOURCES CONFIGURE_DEPENDS ./x86/*.cpp - ${PROJECT_SOURCE_DIR}/include/pernix/x86/*.h ) list(APPEND PERNIX_SOURCES ${PERNIX_X86_SOURCES}) elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_NEON") @@ -24,7 +215,6 @@ elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_NEON") PERNIX_ARM64_NEON_SOURCES CONFIGURE_DEPENDS ./arm64/neon/*.cpp - ${PROJECT_SOURCE_DIR}/include/pernix/arm64/neon/*.h ) list(APPEND PERNIX_SOURCES ${PERNIX_ARM64_NEON_SOURCES}) elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE") @@ -32,7 +222,6 @@ elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE") PERNIX_ARM64_SVE_SOURCES CONFIGURE_DEPENDS ./arm64/sve/*.cpp - ${PROJECT_SOURCE_DIR}/include/pernix/arm64/sve/*.h ) list(APPEND PERNIX_SOURCES ${PERNIX_ARM64_SVE_SOURCES}) elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE2") @@ -40,11 +229,11 @@ elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE2") PERNIX_ARM64_SVE2_SOURCES CONFIGURE_DEPENDS ./arm64/sve2/*.cpp - ${PROJECT_SOURCE_DIR}/include/pernix/arm64/sve2/*.h ) list(APPEND PERNIX_SOURCES ${PERNIX_ARM64_SVE2_SOURCES}) endif () + add_library(pernix SHARED ${PERNIX_SOURCES}) add_library(pernix::pernix ALIAS pernix) set_target_properties(pernix PROPERTIES @@ -53,9 +242,12 @@ set_target_properties(pernix PROPERTIES ) target_compile_features(pernix PUBLIC cxx_std_20) target_compile_options(pernix PRIVATE ${PERNIX_PRIVATE_COMPILE_OPTIONS}) -target_include_directories(pernix PUBLIC +target_include_directories(pernix + PUBLIC $ $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/internal ) if (PERNIX_ENABLE_LTO) set_target_properties(pernix PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) @@ -160,3 +352,5 @@ if (PERNIX_ENABLE_DOXYGEN) DESTINATION ${CMAKE_INSTALL_DOCDIR}) endif () endif () + +]===] \ No newline at end of file diff --git a/src/arm64/neon/compression.cpp b/src/arm64/neon/compression.cpp index 5968f79..81c1cc2 100644 --- a/src/arm64/neon/compression.cpp +++ b/src/arm64/neon/compression.cpp @@ -1,21 +1,28 @@ +#include #include -namespace pernix { -extern "C" { -int neon_compress_block(uint8_t, const float_t*, float_t, uint8_t*) { - return -1; +namespace pernix::internal { +Kernel select_neon_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; } -int neon_compress_block_f64(uint8_t, const double_t*, double_t, uint8_t*) { - return -1; +Kernel select_neon_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; } -int neon_compress_blocks(uint8_t, const float_t*, float_t, uint8_t*, uint32_t) { - return -1; +Kernel select_neon_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; } -int neon_compress_blocks_f64(uint8_t, const double_t*, double_t, uint8_t*, uint32_t) { - return -1; +Kernel select_neon_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; } } -} // namespace pernix diff --git a/src/arm64/neon/decompression.cpp b/src/arm64/neon/decompression.cpp index a89f763..94da2fb 100644 --- a/src/arm64/neon/decompression.cpp +++ b/src/arm64/neon/decompression.cpp @@ -1,143 +1,201 @@ +#include #include -namespace pernix { -extern "C" { -#define PERNIX_NEON_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return arm64::neon::neon_decompress_block(input, scale, output); +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("neon", &arm64::neon::neon_decompress_block); \ + return Kernel("neon", &arm64::neon::neon_decompress_block) -#define PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return arm64::neon::neon_decompress_blocks(input, scale, output, blocks); +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("neon", &arm64::neon::neon_decompress_blocks); \ + return Kernel("neon", &arm64::neon::neon_decompress_blocks) -int neon_decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(1) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(2) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(3) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(4) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(5) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(6) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(7) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(8) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(9) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(10) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(11) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(12) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(13) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(14) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(15) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(16) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(17) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(18) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(19) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(20) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(21) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(22) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(23) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("neon", &arm64::neon::neon_decompress_block); \ + return Kernel("neon", &arm64::neon::neon_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("neon", &arm64::neon::neon_decompress_blocks); \ + return Kernel("neon", &arm64::neon::neon_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"neon", nullptr}; \ + } \ + break + +Kernel select_neon_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"neon", nullptr}; } } -int neon_decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(1) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(2) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(3) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(4) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(5) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(6) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(7) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(8) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(9) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(10) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(11) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(12) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(13) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(14) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(15) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(16) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(17) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(18) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(19) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(20) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(21) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(22) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(23) - PERNIX_NEON_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; +Kernel select_neon_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"neon", nullptr}; } } -int neon_decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - switch (bit_width) { - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; +Kernel select_neon_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"neon", nullptr}; } } -int neon_decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_NEON_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; +Kernel select_neon_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"neon", nullptr}; } } -#undef PERNIX_NEON_DECOMPRESS_BLOCK_CASE -#undef PERNIX_NEON_DECOMPRESS_BLOCKS_CASE +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 } -} // namespace pernix diff --git a/src/arm64/sve/compression.cpp b/src/arm64/sve/compression.cpp deleted file mode 100644 index e973183..0000000 --- a/src/arm64/sve/compression.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include - -namespace pernix { -extern "C" { -int sve_compress_block(uint8_t, const float_t*, float_t, uint8_t*) { - return -1; -} - -int sve_compress_block_f64(uint8_t, const double_t*, double_t, uint8_t*) { - return -1; -} - -int sve_compress_blocks(uint8_t, const float_t*, float_t, uint8_t*, uint32_t) { - return -1; -} - -int sve_compress_blocks_f64(uint8_t, const double_t*, double_t, uint8_t*, uint32_t) { - return -1; -} -} -} // namespace pernix diff --git a/src/arm64/sve/decompression.cpp b/src/arm64/sve/decompression.cpp deleted file mode 100644 index c6d84be..0000000 --- a/src/arm64/sve/decompression.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include - -namespace pernix { -extern "C" { -int sve_decompress_block(uint8_t, const uint8_t*, float_t, float_t*) { - return -1; -} - -int sve_decompress_block_f64(uint8_t, const uint8_t*, double_t, double_t*) { - return -1; -} - -int sve_decompress_blocks(uint8_t, const uint8_t*, float_t, float_t*, uint32_t) { - return -1; -} - -int sve_decompress_blocks_f64(uint8_t, const uint8_t*, double_t, double_t*, uint32_t) { - return -1; -} -} -} // namespace pernix diff --git a/src/arm64/sve2/compression.cpp b/src/arm64/sve2/compression.cpp index 0a55f16..c6d8dd0 100644 --- a/src/arm64/sve2/compression.cpp +++ b/src/arm64/sve2/compression.cpp @@ -1,21 +1,28 @@ +#include #include -namespace pernix { -extern "C" { -int sve2_compress_block(uint8_t, const float_t*, float_t, uint8_t*) { - return -1; +namespace pernix::internal { +Kernel select_sve2_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; } -int sve2_compress_block_f64(uint8_t, const double_t*, double_t, uint8_t*) { - return -1; +Kernel select_sve2_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; } -int sve2_compress_blocks(uint8_t, const float_t*, float_t, uint8_t*, uint32_t) { - return -1; +Kernel select_sve2_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; } -int sve2_compress_blocks_f64(uint8_t, const double_t*, double_t, uint8_t*, uint32_t) { - return -1; +Kernel select_sve2_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; } } -} // namespace pernix diff --git a/src/arm64/sve2/decompression.cpp b/src/arm64/sve2/decompression.cpp index 8429e97..ba796e0 100644 --- a/src/arm64/sve2/decompression.cpp +++ b/src/arm64/sve2/decompression.cpp @@ -1,143 +1,201 @@ +#include #include -namespace pernix { -extern "C" { -#define PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return arm64::sve2::sve2_decompress_block(input, scale, output); +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("sve2", &arm64::sve2::sve2_decompress_block); \ + return Kernel("sve2", &arm64::sve2::sve2_decompress_block) -#define PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return arm64::sve2::sve2_decompress_blocks(input, scale, output, blocks); +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("sve2", &arm64::sve2::sve2_decompress_blocks); \ + return Kernel("sve2", &arm64::sve2::sve2_decompress_blocks) -int sve2_decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(1) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(2) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(3) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(4) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(5) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(6) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(7) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(8) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(9) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(10) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(11) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(12) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(13) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(14) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(15) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(16) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(17) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(18) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(19) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(20) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(21) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(22) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(23) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("sve2", &arm64::sve2::sve2_decompress_block); \ + return Kernel("sve2", &arm64::sve2::sve2_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("sve2", &arm64::sve2::sve2_decompress_blocks); \ + return Kernel("sve2", &arm64::sve2::sve2_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"sve2", nullptr}; \ + } \ + break + +Kernel select_sve2_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"sve2", nullptr}; } } -int sve2_decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(1) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(2) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(3) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(4) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(5) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(6) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(7) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(8) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(9) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(10) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(11) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(12) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(13) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(14) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(15) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(16) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(17) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(18) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(19) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(20) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(21) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(22) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(23) - PERNIX_SVE2_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; +Kernel select_sve2_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"sve2", nullptr}; } } -int sve2_decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - switch (bit_width) { - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; +Kernel select_sve2_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"sve2", nullptr}; } } -int sve2_decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; +Kernel select_sve2_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"sve2", nullptr}; } } -#undef PERNIX_SVE2_DECOMPRESS_BLOCK_CASE -#undef PERNIX_SVE2_DECOMPRESS_BLOCKS_CASE +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 } -} // namespace pernix diff --git a/src/dispatch/cpu_features_arm.cpp b/src/dispatch/cpu_features_arm.cpp new file mode 100644 index 0000000..1b4ac78 --- /dev/null +++ b/src/dispatch/cpu_features_arm.cpp @@ -0,0 +1,23 @@ +#include + +namespace pernix::internal { +CpuFeatures detect_cpu_features() { + CpuFeatures features{}; + + // neon +#if defined(__aarch64__) || defined(_M_ARM64) + features.neon = true; +#elif defined(__ARM_NEON) || defined(__ARM_NEON__) + features.neon = true; +#endif + + // sve +#if defined(__aarch64__) || defined(_M_ARM64) +#ifdef __ARM_FEATURE_SVE + features.sve = true; +#endif +#endif + + return features; +} +} \ No newline at end of file diff --git a/src/dispatch/cpu_features_x86.cpp b/src/dispatch/cpu_features_x86.cpp new file mode 100644 index 0000000..f43245c --- /dev/null +++ b/src/dispatch/cpu_features_x86.cpp @@ -0,0 +1,95 @@ +#include + +#include + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386) || defined(_M_IX86) +#if defined(_MSC_VER) +#include +#else +#include +#include +#endif + + +namespace pernix::internal { +namespace { +#if defined(_MSC_VER) + +void cpuid(int out[4], int leaf, int subleaf) { + __cpuidex(out, leaf, subleaf); +} + +std::uint64_t xgetbv(unsigned int index) { + return _xgetbv(index); +} + +#else + +void cpuid(int out[4], int leaf, int subleaf) { + __cpuid_count(leaf, subleaf, out[0], out[1], out[2], out[3]); +} + +std::uint64_t xgetbv(unsigned int index) { + return _xgetbv(index); +} + +#endif + +bool bit_set(int value, int bit) { + return (value & (1 << bit)) != 0; +} +} // namespace + +CpuFeatures detect_cpu_features() { + CpuFeatures features{}; + + int regs[4]{}; + + cpuid(regs, 1, 0); + + const bool osxsave = bit_set(regs[2], 27); + const bool avx = bit_set(regs[2], 28); + + if (!osxsave || !avx) { + return features; + } + + const std::uint64_t xcr0 = xgetbv(0); + + const bool xmm_enabled = (xcr0 & 0x2) != 0; + const bool ymm_enabled = (xcr0 & 0x4) != 0; + const bool zmm_enabled = + (xcr0 & 0x20) != 0 && + (xcr0 & 0x40) != 0 && + (xcr0 & 0x80) != 0; + + if (!xmm_enabled || !ymm_enabled) { + return features; + } + + cpuid(regs, 7, 0); + + features.avx2 = bit_set(regs[1], 5); + features.bmi2 = bit_set(regs[1], 8); + + if (zmm_enabled) { + features.avx512f = bit_set(regs[1], 16); + features.avx512dq = bit_set(regs[1], 29); + features.avx512bw = bit_set(regs[1], 30); + features.avx512vl = bit_set(regs[1], 31); + features.avx512vbmi = bit_set(regs[2], 1); + } + + return features; +} +} // namespace pernix::internal +#else + +namespace pernix::internal { +CpuFeatures detect_cpu_features() { + return {}; +} +} // namespace pernix::internal + +#endif + diff --git a/src/dispatch/select.cpp b/src/dispatch/select.cpp new file mode 100644 index 0000000..b619af1 --- /dev/null +++ b/src/dispatch/select.cpp @@ -0,0 +1,684 @@ +#include +#include + +namespace pernix::internal { +Kernel select_compress_block_f32(Backend backend, uint8_t bit_width, uint32_t block_size) { + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return select_sve2_compress_block_f32(bit_width, block_size); +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_compress_blocks_f32(Backend backend, uint8_t bit_width, uint32_t block_size) { + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return select_sve2_compress_blocks_f32(bit_width, block_size); +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_compress_block_f64(Backend backend, uint8_t bit_width, uint32_t block_size) { + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return select_sve2_compress_block_f64(bit_width, block_size); +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_compress_blocks_f64(Backend backend, uint8_t bit_width, uint32_t block_size) { + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return select_sve2_compress_blocks_f64(bit_width, block_size); +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_decompress_block_f32(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values) { + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_decompress_blocks_f32(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values) { + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_decompress_block_f64(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values) { + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_decompress_blocks_f64(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values) { + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_auto_compress_block_f32(uint8_t bit_width, uint32_t block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve) { + if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_block_f32(bit_width, block_size); +} + +Kernel select_auto_compress_blocks_f32(uint8_t bit_width, uint32_t block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve) { + if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_blocks_f32(bit_width, block_size); +} + +Kernel select_auto_compress_block_f64(uint8_t bit_width, uint32_t block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve) { + if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_block_f64(bit_width, block_size); +} + +Kernel select_auto_compress_blocks_f64(uint8_t bit_width, uint32_t block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve) { + if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_blocks_f64(bit_width, block_size); +} + +Kernel select_auto_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve) { + if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +} + +Kernel select_auto_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve) { + if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +} + +Kernel select_auto_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve) { + if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +} + +Kernel select_auto_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve) { + if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +} +} diff --git a/src/fallback/compression.cpp b/src/fallback/compression.cpp deleted file mode 100644 index 1ed56e8..0000000 --- a/src/fallback/compression.cpp +++ /dev/null @@ -1,149 +0,0 @@ -#include - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_COMPRESS_BLOCK_CASE(N) \ - case N: \ - return compress_block_fallback(input, scale, output); - -#define PERNIX_COMPRESS_BLOCKS_CASE(N) \ - case N: \ - return compress_blocks_fallback(input, scale, output, blocks); - -int compress_block_fallback(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int compress_block_fallback_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int compress_blocks_fallback(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int compress_blocks_fallback_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus diff --git a/src/fallback/decompression.cpp b/src/fallback/decompression.cpp deleted file mode 100644 index e43b1df..0000000 --- a/src/fallback/decompression.cpp +++ /dev/null @@ -1,150 +0,0 @@ -#include - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return decompress_block_fallback(input, scale, output); - -#define PERNIX_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return decompress_blocks_fallback(input, scale, output, blocks); - -int decompress_block_fallback(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int decompress_block_fallback_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int decompress_blocks_fallback(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int decompress_blocks_fallback_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_DECOMPRESS_BLOCK_CASE -#undef PERNIX_DECOMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus \ No newline at end of file diff --git a/src/fallback/fallback_compression.cpp b/src/fallback/fallback_compression.cpp new file mode 100644 index 0000000..eeaa34d --- /dev/null +++ b/src/fallback/fallback_compression.cpp @@ -0,0 +1,194 @@ +#include +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ +case N: return Kernel("fallback", &compress_block_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ +case N: return Kernel("fallback", &compress_blocks_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ +case N: return Kernel("fallback", &compress_block_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ +case N: return Kernel("fallback", &compress_blocks_fallback) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: return {"fallback", nullptr}; \ + } + +Kernel select_fallback_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +Kernel select_fallback_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +Kernel select_fallback_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +Kernel select_fallback_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} diff --git a/src/fallback/fallback_decompression.cpp b/src/fallback/fallback_decompression.cpp new file mode 100644 index 0000000..232b831 --- /dev/null +++ b/src/fallback/fallback_decompression.cpp @@ -0,0 +1,201 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) + + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"fallback", nullptr}; \ + } \ + break + +Kernel select_fallback_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"fallback", nullptr}; + } +} + +Kernel select_fallback_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"fallback", nullptr}; + } +} + +Kernel select_fallback_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"fallback", nullptr}; + } +} + +Kernel select_fallback_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"fallback", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} diff --git a/src/internal/pernix/arm64/neon/common.h b/src/internal/pernix/arm64/neon/common.h new file mode 100644 index 0000000..2677416 --- /dev/null +++ b/src/internal/pernix/arm64/neon/common.h @@ -0,0 +1,223 @@ +#ifndef PERNIX_ARM64_NEON_COMMON_H +#define PERNIX_ARM64_NEON_COMMON_H + +#include + +#include + +namespace pernix::arm64::neon::internal { + struct float64x2x8_t { + float64x2_t val[8]; + }; + + static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t remaining_elements) { + const uint32_t tail_bits = remaining_elements * bit_width; + const uint32_t tail_bytes = (tail_bits + 7u) / 8u; + return tail_bytes; + } + +__always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(const int8x16_t &input) { + const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); + const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); + + return { + { + vmovl_s16(vget_low_s16(s16_lo)), + vmovl_s16(vget_high_s16(s16_lo)), + vmovl_s16(vget_low_s16(s16_hi)), + vmovl_s16(vget_high_s16(s16_hi)), + } + }; + } + +__always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(const int16x8_t &input) { + return { + { + vmovl_s16(vget_low_s16(input)), + vmovl_s16(vget_high_s16(input)), + } + }; + } + +__always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t &input, const float32x4_t &scale) { + return { + { + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), + } + }; + } + +__always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t &input, const float32x4_t &scale) { + return { + { + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + } + }; + } + +__always_inline float32x4_t neon_dequantize_epi32(const int32x4_t &input, const float32x4_t &scale) { + return vmulq_f32(vcvtq_f32_s32(input), scale); + } + +__always_inline float64x2_t neon_dequantize_epi32_f64(const int32x2_t &input, const float64x2_t &scale) { + return vmulq_f64(vcvtq_f64_s64(vmovl_s32(input)), scale); + } + +__always_inline float64x2x2_t neon_dequantize_epi32_f64(const int32x4_t &input, const float64x2_t &scale) { + return { + { + neon_dequantize_epi32_f64(vget_low_s32(input), scale), + neon_dequantize_epi32_f64(vget_high_s32(input), scale), + } + }; + } + +__always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t &input, const float64x2_t &scale) { + const float64x2x2_t dequantized_low = neon_dequantize_epi32_f64(input.val[0], scale); + const float64x2x2_t dequantized_high = neon_dequantize_epi32_f64(input.val[1], scale); + + return { + { + dequantized_low.val[0], + dequantized_low.val[1], + dequantized_high.val[0], + dequantized_high.val[1], + } + }; + } + +__always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t &input, const float64x2_t &scale) { + const float64x2x2_t dequantized0 = neon_dequantize_epi32_f64(input.val[0], scale); + const float64x2x2_t dequantized1 = neon_dequantize_epi32_f64(input.val[1], scale); + const float64x2x2_t dequantized2 = neon_dequantize_epi32_f64(input.val[2], scale); + const float64x2x2_t dequantized3 = neon_dequantize_epi32_f64(input.val[3], scale); + + return { + { + dequantized0.val[0], + dequantized0.val[1], + dequantized1.val[0], + dequantized1.val[1], + dequantized2.val[0], + dequantized2.val[1], + dequantized3.val[0], + dequantized3.val[1], + } + }; + } + +__always_inline uint8x16_t neon_load_tail_elements_int8(const uint8_t *input, const uint32_t tail_bytes_count) { + uint8_t buffer[16] = {0}; + std::memcpy(buffer, input, tail_bytes_count); + return vld1q_u8(buffer); + } + +__always_inline uint16x8_t neon_load_tail_elements_int16(const uint8_t *input, const uint32_t tail_bytes_count) { + uint16_t buffer[8] = {0}; + std::memcpy(buffer, input, tail_bytes_count); + return vld1q_u16(buffer); + } + +__always_inline uint32x4_t neon_load_tail_elements_int32(const uint8_t *input, const uint32_t tail_bytes_count) { + uint32_t buffer[4] = {0}; + std::memcpy(buffer, input, tail_bytes_count); + return vld1q_u32(buffer); + } + +__always_inline float32x4_t neon_load_tail_elements_f32(const uint8_t *input, const uint32_t tail_elements) { + float32_t buffer[4] = {0.0f}; + std::memcpy(buffer, input, tail_elements * sizeof(float32_t)); + return vld1q_f32(buffer); + } + +__always_inline float64x2_t neon_load_tail_elements_f64(const uint8_t *input, const uint32_t tail_elements) { + float64_t buffer[2] = {0.0}; + std::memcpy(buffer, input, tail_elements * sizeof(float64_t)); + return vld1q_f64(buffer); + } + +__always_inline void neon_store_tail_elements_int8(uint8_t *output, const uint8x16x4_t &data, + const uint32_t tail_elements) { + uint8_t buffer[16 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_u8(buffer + i * 16, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(uint8_t)); + } + +__always_inline void neon_store_tail_elements_int16(uint16_t *output, const uint16x8x4_t &data, + const uint32_t tail_elements) { + uint16_t buffer[8 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_u16(buffer + i * 8, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(uint16_t)); + } + +__always_inline void neon_store_tail_elements_int32(uint32_t *output, const uint32x4x4_t &data, + const uint32_t tail_elements) { + uint32_t buffer[4 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_u32(buffer + i * 4, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(uint32_t)); + } + +__always_inline void neon_store_tail_elements_f32(float32_t *output, const float32x4x4_t &data, + const uint32_t tail_elements) { + float32_t buffer[16 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_f32(buffer + i * 4, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); + } + +__always_inline void neon_store_tail_elements_f32(float32_t *output, const float32x4x2_t &data, + const uint32_t tail_elements) { + float32_t buffer[8 * 2]; + for (uint32_t i = 0; i < 2; ++i) { + vst1q_f32(buffer + i * 4, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); + } + +__always_inline void neon_store_tail_elements_f32(float32_t *output, const float32x4_t &data, + const uint32_t tail_elements) { + float32_t buffer[4]; + vst1q_f32(buffer, data); + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); + } + +__always_inline void neon_store_tail_elements_f64(float64_t *output, const float64x2x4_t &data, + const uint32_t tail_elements) { + float64_t buffer[2 * 4]; + for (uint32_t i = 0; i < 4; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); + } + +__always_inline void neon_store_tail_elements_f64(float64_t *output, const float64x2x2_t &data, + const uint32_t tail_elements) { + float64_t buffer[2 * 2]; + for (uint32_t i = 0; i < 2; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); + } + +__always_inline void neon_store_tail_elements_f64(float64_t *output, const float64x2x8_t &data, + const uint32_t tail_elements) { + float64_t buffer[2 * 8]; + for (uint32_t i = 0; i < 8; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); + } +} // namespace pernix::arm64::neon::internal + +#endif // PERNIX_ARM64_NEON_COMMON_H diff --git a/include/pernix/arm64/neon/compression.h b/src/internal/pernix/arm64/neon/compression.h similarity index 79% rename from include/pernix/arm64/neon/compression.h rename to src/internal/pernix/arm64/neon/compression.h index 6e49348..f88fbb6 100644 --- a/include/pernix/arm64/neon/compression.h +++ b/src/internal/pernix/arm64/neon/compression.h @@ -9,7 +9,7 @@ namespace pernix::arm64::neon { namespace internal { -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_compress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { @@ -17,7 +17,7 @@ __always_inline int neon_compress_block_1to8(const uint8_t* __restrict__ input, return -1; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_compress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { @@ -25,7 +25,7 @@ __always_inline int neon_compress_block_9to16(const uint8_t* __restrict__ input, return -1; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_compress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { @@ -33,7 +33,7 @@ __always_inline int neon_compress_block_17to24(const uint8_t* __restrict__ input return -1; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_compress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { @@ -41,7 +41,7 @@ __always_inline int neon_compress_block_1to8(const uint8_t* __restrict__ input, return -1; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_compress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { @@ -49,7 +49,7 @@ __always_inline int neon_compress_block_9to16(const uint8_t* __restrict__ input, return -1; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_compress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { @@ -58,7 +58,7 @@ __always_inline int neon_compress_block_17to24(const uint8_t* __restrict__ input } } // namespace internal -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_compress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { @@ -72,7 +72,7 @@ __always_inline int neon_compress_block(const uint8_t* __restrict__ input, const return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int neon_compress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { @@ -86,7 +86,7 @@ __always_inline int neon_compress_block(const uint8_t* __restrict__ input, const return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int neon_compress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { @@ -102,7 +102,7 @@ int neon_compress_blocks(const uint8_t* __restrict__ input, const float_t scale, return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int neon_compress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { @@ -116,26 +116,6 @@ int neon_compress_blocks(const uint8_t* __restrict__ input, const double_t scale } return 0; } - -#ifdef __cplusplus -extern "C" { -#endif - -int neon_compress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - - -int neon_compress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output); - -int neon_compress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -int neon_compress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output, uint32_t blocks); - -#ifdef __cplusplus -} -#endif } // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_COMPRESSION_H diff --git a/include/pernix/arm64/neon/decompression.h b/src/internal/pernix/arm64/neon/decompression.h similarity index 77% rename from include/pernix/arm64/neon/decompression.h rename to src/internal/pernix/arm64/neon/decompression.h index 583948f..375adc2 100644 --- a/include/pernix/arm64/neon/decompression.h +++ b/src/internal/pernix/arm64/neon/decompression.h @@ -10,9 +10,10 @@ namespace pernix::arm64::neon { namespace internal { -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_16 = elements_per_block / 16; @@ -22,7 +23,9 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input for (uint32_t i = 0; i < iterations_16; ++i) { const uint8x16_t source = vld1q_u8(input); - const int8x16_t unpacked = b128::neon_unpack_epi8_1to8(source); + const int8x16_t unpacked = b128::neon_unpack_epi8_1to8 + (source); const int32x4x4_t converted = neon_convert_int8x16_int32x4x4(unpacked); const float32x4x4_t dequantized = neon_dequantize_epi32(converted, scale_v); @@ -36,8 +39,11 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input } if constexpr (remaining_elements > 0) { - const uint8x16_t tail_source = neon_load_tail_elements_int8(input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8(tail_source); + const uint8x16_t tail_source = neon_load_tail_elements_int8( + input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8 + (tail_source); const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x4(tail_unpacked); const float32x4x4_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); @@ -48,9 +54,10 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; @@ -60,7 +67,9 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu for (uint32_t i = 0; i < iterations_8; ++i) { const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); - const int16x8_t unpacked = b128::neon_unpack_epi16_9to16(source); + const int16x8_t unpacked = b128::neon_unpack_epi16_9to16 + (source); const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); const float32x4x2_t dequantized = neon_dequantize_epi32(converted, scale_v); @@ -74,8 +83,11 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu } if constexpr (remaining_elements > 0) { - const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16(tail_source); + const uint16x8_t tail_source = neon_load_tail_elements_int16( + input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16 + (tail_source); const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); const float32x4x2_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); @@ -86,9 +98,10 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_4 = elements_per_block / 4; @@ -131,7 +144,8 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp if constexpr (tail_bit_offset == 0) { tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } else { - tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); + tail_unpacked = b128::neon_unpack_epi32_17to24( + tail_source); } const float32x4_t tail_dequantized = neon_dequantize_epi32(tail_unpacked, scale_v); @@ -142,9 +156,10 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_16 = elements_per_block / 16; @@ -154,7 +169,9 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input for (uint32_t i = 0; i < iterations_16; ++i) { const uint8x16_t source = vld1q_u8(input); - const int8x16_t unpacked = b128::neon_unpack_epi8_1to8(source); + const int8x16_t unpacked = b128::neon_unpack_epi8_1to8 + (source); const int32x4x4_t converted = neon_convert_int8x16_int32x4x4(unpacked); const float64x2x8_t dequantized = neon_dequantize_epi32_f64(converted, scale_v); @@ -168,8 +185,11 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input } if constexpr (remaining_elements > 0) { - const uint8x16_t tail_source = neon_load_tail_elements_int8(input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8(tail_source); + const uint8x16_t tail_source = neon_load_tail_elements_int8( + input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8 + (tail_source); const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x4(tail_unpacked); const float64x2x8_t tail_dequantized = neon_dequantize_epi32_f64(tail_converted, scale_v); @@ -180,9 +200,10 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; @@ -192,7 +213,9 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu for (uint32_t i = 0; i < iterations_8; ++i) { const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); - const int16x8_t unpacked = b128::neon_unpack_epi16_9to16(source); + const int16x8_t unpacked = b128::neon_unpack_epi16_9to16 + (source); const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); const float64x2x4_t dequantized = neon_dequantize_epi32_f64(converted, scale_v); @@ -206,8 +229,11 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu } if constexpr (remaining_elements > 0) { - const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16(tail_source); + const uint16x8_t tail_source = neon_load_tail_elements_int16( + input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16 + (tail_source); const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); const float64x2x4_t tail_dequantized = neon_dequantize_epi32_f64(tail_converted, scale_v); @@ -218,9 +244,10 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_4 = elements_per_block / 4; @@ -264,7 +291,8 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp if constexpr (tail_bit_offset == 0) { tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } else { - tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); + tail_unpacked = b128::neon_unpack_epi32_17to24( + tail_source); } const float64x2x2_t tail_dequantized = neon_dequantize_epi32_f64(tail_unpacked, scale_v); @@ -276,37 +304,46 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp } } // namespace internal -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +__always_inline int neon_decompress_block(const void* __restrict__ input, const float_t scale, + void* __restrict__ output) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::neon_decompress_block_1to8(input, scale, output); + return internal::neon_decompress_block_1to8(typed_input, scale, typed_output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::neon_decompress_block_9to16(input, scale, output); + return internal::neon_decompress_block_9to16(typed_input, scale, typed_output); } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::neon_decompress_block_17to24(input, scale, output); + return internal::neon_decompress_block_17to24(typed_input, scale, typed_output); } return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +__always_inline int neon_decompress_block(const void* __restrict__ input, const double_t scale, + void* __restrict__ output) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::neon_decompress_block_1to8(input, scale, output); + return internal::neon_decompress_block_1to8(typed_input, scale, typed_output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::neon_decompress_block_9to16(input, scale, output); + return internal::neon_decompress_block_9to16(typed_input, scale, typed_output); } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::neon_decompress_block_17to24(input, scale, output); + return internal::neon_decompress_block_17to24(typed_input, scale, typed_output); } return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; +int neon_decompress_blocks(const void* __restrict__ input, const float_t scale, void* __restrict__ output, + const uint32_t blocks) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + const uint8_t* block_input = typed_input; + float_t* block_output = typed_output; for (uint32_t block = 0; block < blocks; ++block) { neon_decompress_block(block_input, scale, block_output); @@ -317,37 +354,23 @@ int neon_decompress_blocks(const uint8_t* __restrict__ input, const float_t scal return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; +int neon_decompress_blocks(const void* __restrict__ input, const double_t scale, void* __restrict__ output, + const uint32_t blocks) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + const uint8_t* block_input = typed_input; + double_t* block_output = typed_output; for (uint32_t block = 0; block < blocks; ++block) { neon_decompress_block(block_input, scale, block_output); block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } - return 0; -} - -#ifdef __cplusplus -extern "C" { -#endif - -int neon_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); -int neon_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -int neon_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -int neon_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus + return 0; } -#endif } // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_DECOMPRESSION_H diff --git a/include/pernix/arm64/neon/packing.h b/src/internal/pernix/arm64/neon/packing.h similarity index 100% rename from include/pernix/arm64/neon/packing.h rename to src/internal/pernix/arm64/neon/packing.h diff --git a/include/pernix/arm64/neon/tables.h b/src/internal/pernix/arm64/neon/tables.h similarity index 100% rename from include/pernix/arm64/neon/tables.h rename to src/internal/pernix/arm64/neon/tables.h diff --git a/src/internal/pernix/arm64/neon/unpacking.h b/src/internal/pernix/arm64/neon/unpacking.h new file mode 100644 index 0000000..e70fbbc --- /dev/null +++ b/src/internal/pernix/arm64/neon/unpacking.h @@ -0,0 +1,101 @@ +#ifndef PERNIX_ARM64_NEON_UNPACKING_H +#define PERNIX_ARM64_NEON_UNPACKING_H + +#include +#include + +using namespace pernix::arm64::neon::internal; + +namespace pernix::arm64::neon::internal::b128 { + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t &input) { + if constexpr (BIT_WIDTH == 8) { + return vreinterpretq_s8_u8(input); + } else if constexpr (BIT_WIDTH == 1) { + using tables = table_unpacking; + + const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); + const uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); + + return vreinterpretq_s8_u8(vandq_u8(shifted, vdupq_n_u8(1))); + } else { + using tables = table_unpacking; + + const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); + + uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); + + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + const uint8x16_t permuted2_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute2.data())); + + shifted = vorrq_u8(shifted, vshlq_u8(permuted2_u8, vld1q_s8(tables::shift2.data()))); + } + + constexpr int shift = 8 - BIT_WIDTH; + shifted = vshlq_n_u8(shifted, shift); + + if constexpr (SIGN_VALUES) { + return vshlq_s8(vreinterpretq_s8_u8(shifted), vdupq_n_s8(-shift)); + } else { + return vreinterpretq_s8_u8(vshlq_u8(shifted, vdupq_n_s8(-shift))); + } + } + } + + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t &input) { + if constexpr (BIT_WIDTH == 16) { + return vreinterpretq_s16_u16(input); + } else { + using tables = table_unpacking; + + const uint8x16_t input_u8 = vreinterpretq_u8_u16(input); + + const uint8x16_t permuted1_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute1.data())); + + uint16x8_t shifted = vshlq_u16(vreinterpretq_u16_u8(permuted1_u8), vld1q_s16(tables::shift1.data())); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const uint8x16_t permuted2_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute2.data())); + + const uint16x8_t shifted2 = vshlq_u16(vreinterpretq_u16_u8(permuted2_u8), + vld1q_s16(tables::shift2.data())); + + shifted = vorrq_u16(shifted, shifted2); + } + + constexpr int shift = 16 - BIT_WIDTH; + shifted = vshlq_n_u16(shifted, shift); + + if constexpr (SIGN_VALUES) { + return vshlq_s16(vreinterpretq_s16_u16(shifted), vdupq_n_s16(-shift)); + } else { + return vreinterpretq_s16_u16(vshlq_u16(shifted, vdupq_n_s16(-shift))); + } + } + } + + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline int32x4_t neon_unpack_epi32_17to24(const uint32x4_t &input) { + using tables = table_unpacking; + + const uint8x16_t input_8 = vreinterpretq_u8_u32(input); + + const uint8x16_t permuted_u8 = vqtbl1q_u8(input_8, vld1q_u8(tables::permute.data())); + + const uint32x4_t value = vshlq_u32(vreinterpretq_u32_u8(permuted_u8), vld1q_s32(tables::shift.data())); + + if constexpr (SIGN_VALUES) { + constexpr int sign_shift = 32 - BIT_WIDTH; + return vshrq_n_s32(vreinterpretq_s32_u32(vshlq_n_u32(value, sign_shift)), sign_shift); + } else { + constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1u; + return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); + } + } +} // namespace pernix::arm64::neon::internal::b128 + +#endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/arm64/sve2/compression.h b/src/internal/pernix/arm64/sve2/compression.h similarity index 52% rename from include/pernix/arm64/sve2/compression.h rename to src/internal/pernix/arm64/sve2/compression.h index 4e4627d..72d9229 100644 --- a/include/pernix/arm64/sve2/compression.h +++ b/src/internal/pernix/arm64/sve2/compression.h @@ -12,48 +12,37 @@ template inline constexpr bool sve2_compression_unimplemented_v = false; } // namespace internal -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_compress_block(const float_t*, float_t, uint8_t*) { - static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); return -1; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_compress_block(const double_t*, double_t, uint8_t*) { - static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); return -1; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_compress_blocks(const float_t*, float_t, uint8_t*, uint32_t) { - static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); return -1; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_compress_blocks(const double_t*, double_t, uint8_t*, uint32_t) { - static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); return -1; } - -#ifdef __cplusplus -extern "C" { -#endif - -int sve2_compress_block(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); -int sve2_compress_block_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); -int sve2_compress_blocks(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, - uint32_t blocks); -int sve2_compress_blocks_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -#endif } // namespace pernix #endif // PERNIX_ARM64_SVE2_COMPRESSION_H diff --git a/include/pernix/arm64/sve2/decompression.h b/src/internal/pernix/arm64/sve2/decompression.h similarity index 76% rename from include/pernix/arm64/sve2/decompression.h rename to src/internal/pernix/arm64/sve2/decompression.h index 2128ff2..350a196 100644 --- a/include/pernix/arm64/sve2/decompression.h +++ b/src/internal/pernix/arm64/sve2/decompression.h @@ -17,13 +17,15 @@ template return (elements * BIT_WIDTH + 7) / 8; } -[[nodiscard]] __always_inline svuint8_t sve2_load_packed_bytes(const uint8_t* __restrict__ input, const uint32_t bytes) { +[[nodiscard]] __always_inline svuint8_t sve2_load_packed_bytes(const uint8_t* __restrict__ input, + const uint32_t bytes) { const svbool_t pg = svwhilelt_b8(uint64_t{0}, static_cast(bytes)); return svld1_u8(pg, input); } template -__always_inline void sve2_store_dequantized_i8_f32(svint8_t values, const svfloat32_t scale_v, float_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i8_f32(svint8_t values, const svfloat32_t scale_v, + float_t* __restrict__ output, const uint32_t count) { alignas(64) std::vector temp(svcntb()); @@ -50,7 +52,8 @@ __always_inline void sve2_store_dequantized_i8_f32(svint8_t values, const svfloa } template -__always_inline void sve2_store_dequantized_i8_f64(svint8_t values, const double_t scale, double_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i8_f64(svint8_t values, const double_t scale, + double_t* __restrict__ output, const uint32_t count) { std::vector temp(svcntb()); @@ -66,7 +69,8 @@ __always_inline void sve2_store_dequantized_i8_f64(svint8_t values, const double } template -__always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfloat32_t scale_v, float_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfloat32_t scale_v, + float_t* __restrict__ output, const uint32_t count) { alignas(64) std::vector temp(svcnth()); @@ -81,8 +85,9 @@ __always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfl const svint32_t widened = svld1sh_s32(pg, temp.data() + offset); dequantized = svmul_f32_x(pg, svcvt_f32_s32_x(pg, widened), scale_v); } else { - const svuint32_t widened = svld1uh_u32(pg, reinterpret_cast(temp.data() + offset)); - dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, widened), scale_v); + const svuint32_t widened = + svld1uh_u32(pg, reinterpret_cast(temp.data() + offset)); + dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, widened), scale_v); } svst1_f32(pg, output + offset, dequantized); @@ -92,7 +97,8 @@ __always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfl } template -__always_inline void sve2_store_dequantized_i16_f64(svint16_t values, const double_t scale, double_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i16_f64(svint16_t values, const double_t scale, + double_t* __restrict__ output, const uint32_t count) { std::vector temp(svcnth()); @@ -108,7 +114,8 @@ __always_inline void sve2_store_dequantized_i16_f64(svint16_t values, const doub } template -__always_inline void sve2_store_dequantized_i32_f32(svint32_t values, const svfloat32_t scale_v, float_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i32_f32(svint32_t values, const svfloat32_t scale_v, + float_t* __restrict__ output, const uint32_t count) { const svbool_t pg = svwhilelt_b32(uint64_t{0}, static_cast(count)); @@ -123,7 +130,8 @@ __always_inline void sve2_store_dequantized_i32_f32(svint32_t values, const svfl } template -__always_inline void sve2_store_dequantized_i32_f64(svint32_t values, const double_t scale, double_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i32_f64(svint32_t values, const double_t scale, + double_t* __restrict__ output, const uint32_t count) { std::vector temp(svcntw()); @@ -138,9 +146,10 @@ __always_inline void sve2_store_dequantized_i32_f64(svint32_t values, const doub } } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcntb()); @@ -166,20 +175,23 @@ __always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input const uint8_t* chunk_input = input + input_bit_offset / 8; const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); - const svint8_t unpacked = sve2_unpack_epi8_1to8(source, permute, shift, spill_permute, spill_shift); + const svint8_t unpacked = sve2_unpack_epi8_1to8 + (source, permute, shift, spill_permute, spill_shift); sve2_store_dequantized_i8_f32(unpacked, scale_v, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcnth()); @@ -204,22 +216,25 @@ __always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ inpu const uint32_t bytes = packed_bytes(count); const uint8_t* chunk_input = input + input_bit_offset / 8; - const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); const svint16_t unpacked = - sve2_unpack_epi16_9to16(svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); + sve2_unpack_epi16_9to16 + (svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); sve2_store_dequantized_i16_f32(unpacked, scale_v, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, + float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcntw()); @@ -246,15 +261,16 @@ __always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ inp sve2_store_dequantized_i32_f32(unpacked, scale_v, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcntb()); @@ -278,20 +294,23 @@ __always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input const uint8_t* chunk_input = input + input_bit_offset / 8; const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); - const svint8_t unpacked = sve2_unpack_epi8_1to8(source, permute, shift, spill_permute, spill_shift); + const svint8_t unpacked = sve2_unpack_epi8_1to8 + (source, permute, shift, spill_permute, spill_shift); sve2_store_dequantized_i8_f64(unpacked, scale, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcnth()); @@ -314,22 +333,25 @@ __always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ inpu const uint32_t bytes = packed_bytes(count); const uint8_t* chunk_input = input + input_bit_offset / 8; - const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); const svint16_t unpacked = - sve2_unpack_epi16_9to16(svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); + sve2_unpack_epi16_9to16 + (svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); sve2_store_dequantized_i16_f64(unpacked, scale, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, + double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcntw()); @@ -354,84 +376,77 @@ __always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ inp sve2_store_dequantized_i32_f64(unpacked, scale, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; } -} // namespace internal +} // namespace internal -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +int sve2_decompress_block(const void* __restrict__ input, const float_t scale, void* __restrict__ output) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::sve2_decompress_block_1to8(input, scale, output); + return internal::sve2_decompress_block_1to8(typed_input, scale, typed_output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::sve2_decompress_block_9to16(input, scale, output); + return internal::sve2_decompress_block_9to16(typed_input, scale, typed_output); } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::sve2_decompress_block_17to24(input, scale, output); + return internal::sve2_decompress_block_17to24(typed_input, scale, typed_output); } } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +int sve2_decompress_block(const void* __restrict__ input, const double_t scale, + void* __restrict__ output) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::sve2_decompress_block_1to8(input, scale, output); + return internal::sve2_decompress_block_1to8(typed_input, scale, typed_output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::sve2_decompress_block_9to16(input, scale, output); + return internal::sve2_decompress_block_9to16(typed_input, scale, typed_output); } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::sve2_decompress_block_17to24(input, scale, output); + return internal::sve2_decompress_block_17to24(typed_input, scale, typed_output); } } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; +int sve2_decompress_blocks(const void* __restrict__ input, const float_t scale, void* __restrict__ output, + const uint32_t blocks) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + const uint8_t* block_input = typed_input; + float_t* block_output = typed_output; for (uint32_t block = 0; block < blocks; ++block) { sve2_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; +int sve2_decompress_blocks(const void* __restrict__ input, const double_t scale, void* __restrict__ output, + const uint32_t blocks) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + const uint8_t* block_input = typed_input; + double_t* block_output = typed_output; for (uint32_t block = 0; block < blocks; ++block) { sve2_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } - -#ifdef __cplusplus -extern "C" { -#endif - -int sve2_decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -int sve2_decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -int sve2_decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -int sve2_decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -#endif -} // namespace pernix::arm64::sve2 +} // namespace pernix::arm64::sve2 #endif // PERNIX_ARM64_SVE2_DECOMPRESSION_H diff --git a/include/pernix/arm64/sve2/packing.h b/src/internal/pernix/arm64/sve2/packing.h similarity index 74% rename from include/pernix/arm64/sve2/packing.h rename to src/internal/pernix/arm64/sve2/packing.h index 789b4d7..5cf2355 100644 --- a/include/pernix/arm64/sve2/packing.h +++ b/src/internal/pernix/arm64/sve2/packing.h @@ -4,8 +4,8 @@ #include namespace pernix::arm64::sve2::internal { -template -inline constexpr bool packing_unimplemented_v = false; + template + inline constexpr bool packing_unimplemented_v = false; } // namespace pernix::arm64::sve2::internal #endif // PERNIX_ARM64_SVE2_PACKING_H diff --git a/src/internal/pernix/arm64/sve2/tables.h b/src/internal/pernix/arm64/sve2/tables.h new file mode 100644 index 0000000..8ef61d8 --- /dev/null +++ b/src/internal/pernix/arm64/sve2/tables.h @@ -0,0 +1,119 @@ +#ifndef PERNIX_ARM64_SVE2_TABLES_H +#define PERNIX_ARM64_SVE2_TABLES_H + +#include + +#include + +namespace pernix::arm64::sve2::internal { + template + struct table_unpacking { + static constexpr uint8_t bit_width = BIT_WIDTH; + + static svbool_t pg_b8() { return svptrue_b8(); } + + static svbool_t pg_b16() { return svptrue_b16(); } + + static svbool_t pg_b32() { return svptrue_b32(); } + }; + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) + struct table_unpacking { + static constexpr uint8_t bit_width = BIT_WIDTH; + + static svuint8_t permute() { + const svbool_t pg = svptrue_b8(); + return svlsr_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 3); + } + + static svuint8_t spill_permute() { + const svbool_t pg = svptrue_b8(); + return svadd_n_u8_x(pg, permute(), 1); + } + + static svuint8_t shift() { + const svbool_t pg = svptrue_b8(); + return svand_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 7); + } + + static svuint8_t spill_shift() { + const svbool_t pg = svptrue_b8(); + return svsub_u8_x(pg, svdup_n_u8(8), shift()); + } + }; + + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) + struct table_unpacking { + static constexpr uint8_t bit_width = BIT_WIDTH; + + static svuint8_t permute() { + const svbool_t pg = svptrue_b8(); + const svuint8_t lane = svindex_u8(0, 1); + const svuint8_t elem = svlsr_n_u8_x(pg, lane, 1); + const svuint8_t byte = svand_n_u8_x(pg, lane, 1); + + svuint8_t first; + if constexpr (BIT_WIDTH == 16) { + first = svlsl_n_u8_x(pg, elem, 1); + } else { + constexpr uint8_t extra_bits = BIT_WIDTH - 8u; + const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); + const svuint8_t low = svlsr_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), 3); + first = svadd_u8_x(pg, elem, svadd_u8_x(pg, high, low)); + } + + return svadd_u8_x(pg, first, byte); + } + + static svuint8_t spill_permute() { + const svbool_t pg = svptrue_b8(); + return svadd_n_u8_x(pg, permute(), 2); + } + + static svuint16_t shift() { + const svbool_t pg = svptrue_b16(); + return svand_n_u16_x(pg, svmul_n_u16_x(pg, svindex_u16(0, 1), BIT_WIDTH), 7); + } + + static svuint16_t spill_shift() { + const svbool_t pg = svptrue_b16(); + const svuint16_t bit_shift = shift(); + const svuint16_t spill = svsub_u16_x(pg, svdup_n_u16(16), bit_shift); + return svsel_u16(svcmpgt_n_u16(pg, bit_shift, 16u - BIT_WIDTH), spill, svdup_n_u16(16)); + } + }; + + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && START_BIT_OFFSET < 8) + struct table_unpacking { + static constexpr uint8_t bit_width = BIT_WIDTH; + + static svuint8_t permute() { + const svbool_t pg = svptrue_b8(); + const svuint8_t lane = svindex_u8(0, 1); + const svuint8_t elem = svlsr_n_u8_x(pg, lane, 2); + const svuint8_t byte = svand_n_u8_x(pg, lane, 3); + + svuint8_t first = svmul_n_u8_x(pg, elem, BIT_WIDTH / 8u); + if constexpr (BIT_WIDTH % 8u != 0) { + constexpr uint8_t extra_bits = BIT_WIDTH % 8u; + const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); + const svuint8_t low_bits = + svadd_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), START_BIT_OFFSET); + first = svadd_u8_x(pg, first, svadd_u8_x(pg, high, svlsr_n_u8_x(pg, low_bits, 3))); + } + + return svadd_u8_x(pg, first, byte); + } + + static svuint32_t shift() { + const svbool_t pg = svptrue_b32(); + return svand_n_u32_x(pg, svadd_n_u32_x(pg, svmul_n_u32_x(pg, svindex_u32(0, 1), BIT_WIDTH), + START_BIT_OFFSET), 7); + } + }; +} // namespace pernix::arm64::sve2::internal + +#endif // PERNIX_ARM64_SVE2_TABLES_H diff --git a/src/internal/pernix/arm64/sve2/unpacking.h b/src/internal/pernix/arm64/sve2/unpacking.h new file mode 100644 index 0000000..8bb7e3c --- /dev/null +++ b/src/internal/pernix/arm64/sve2/unpacking.h @@ -0,0 +1,94 @@ +#ifndef PERNIX_ARM64_SVE2_UNPACKING_H +#define PERNIX_ARM64_SVE2_UNPACKING_H + +#include + +#include "tables.h" + +namespace pernix::arm64::sve2::internal { + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline svint8_t sve2_unpack_epi8_1to8(const svuint8_t input, const svuint8_t permute, const svuint8_t shift, + const svuint8_t spill_permute, const svuint8_t spill_shift) { + if constexpr (BIT_WIDTH == 8) { + return svreinterpret_s8(input); + } else { + const svbool_t pg = svptrue_b8(); + + const svuint8_t permuted = svtbl_u8(input, permute); + svuint8_t unpacked = svlsr_u8_x(pg, permuted, shift); + + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + const svuint8_t spill_permuted_values = svtbl_u8(input, spill_permute); + const svuint8_t spill_shifted = svlsl_u8_x(pg, spill_permuted_values, spill_shift); + unpacked = svorr_u8_x(pg, unpacked, spill_shifted); + } + + if constexpr (BIT_WIDTH == 1) { + unpacked = svand_n_u8_x(pg, unpacked, 1); + return svreinterpret_s8(unpacked); + } else { + constexpr int sign_shift = 8 - BIT_WIDTH; + + unpacked = svlsl_n_u8_x(pg, unpacked, sign_shift); + + if constexpr (SIGN_VALUES) { + return svasr_n_s8_x(pg, svreinterpret_s8_u8(unpacked), sign_shift); + } else { + return svreinterpret_s8_u8(svlsr_n_u8_x(pg, unpacked, sign_shift)); + } + } + } + } + + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline svint16_t sve2_unpack_epi16_9to16(const svuint16_t input, const svuint8_t permute, + const svuint16_t shift, + const svuint8_t spill_permute, const svuint16_t spill_shift) { + if constexpr (BIT_WIDTH == 16) { + return svreinterpret_s16(input); + } else { + const svbool_t pg = svptrue_b16(); + + const svuint8_t permuted = svtbl_u8(svreinterpret_u8_u16(input), permute); + svuint16_t shifted = svlsr_u16_x(pg, svreinterpret_u16_u8(permuted), shift); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const svuint8_t spill_permuted_values = svtbl_u8(svreinterpret_u8_u16(input), spill_permute); + const svuint16_t spill_shifted = svlsl_u16_x(pg, svreinterpret_u16_u8(spill_permuted_values), + spill_shift); + shifted = svorr_u16_x(pg, shifted, spill_shifted); + } + + constexpr int sign_shift = 16 - BIT_WIDTH; + shifted = svlsl_n_u16_x(pg, shifted, sign_shift); + + if constexpr (SIGN_VALUES) { + return svasr_n_s16_x(pg, svreinterpret_s16_u16(shifted), sign_shift); + } else { + return svreinterpret_s16_u16(svlsr_n_u16_x(pg, shifted, sign_shift)); + } + } + } + + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline svint32_t sve2_unpack_epi32_17to24(const svuint8_t input) { + using table = table_unpacking; + + const svbool_t pg = svptrue_b32(); + const svuint8_t permuted = svtbl_u8(input, table::permute()); + const svuint32_t unpacked = svlsr_u32_x(pg, svreinterpret_u32_u8(permuted), table::shift()); + + if constexpr (SIGN_VALUES) { + constexpr int sign_shift = 32 - BIT_WIDTH; + return svasr_n_s32_x(pg, svreinterpret_s32_u32(svlsl_n_u32_x(pg, unpacked, sign_shift)), sign_shift); + } else { + constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1u; + return svreinterpret_s32_u32(svand_n_u32_x(pg, unpacked, mask)); + } + } +} // namespace pernix::arm64::sve2::internal + +#endif // PERNIX_ARM64_SVE2_UNPACKING_H diff --git a/src/internal/pernix/dispatch/cpu_features.h b/src/internal/pernix/dispatch/cpu_features.h new file mode 100644 index 0000000..bcf3e7d --- /dev/null +++ b/src/internal/pernix/dispatch/cpu_features.h @@ -0,0 +1,26 @@ +#ifndef PERNIX_CPU_FEATURES_H +#define PERNIX_CPU_FEATURES_H + +namespace pernix::internal { +struct CpuFeatures { + bool avx2 = false; + bool bmi2 = false; + bool avx512f = false; + bool avx512dq = false; + bool avx512bw = false; + bool avx512vl = false; + bool avx512vbmi = false; + bool neon = false; + bool sve = false; + bool sve2 = false; +}; + +CpuFeatures detect_cpu_features(); + +inline CpuFeatures get_cached_cpu_features() { + static const CpuFeatures features = detect_cpu_features(); + return features; +} +} + +#endif //PERNIX_CPU_FEATURES_H diff --git a/src/internal/pernix/dispatch/kernel.h b/src/internal/pernix/dispatch/kernel.h new file mode 100644 index 0000000..1d51c64 --- /dev/null +++ b/src/internal/pernix/dispatch/kernel.h @@ -0,0 +1,27 @@ +#ifndef PERNIX_KERNEL_H +#define PERNIX_KERNEL_H + +#include +#include + +namespace pernix::internal { +using KernelBlockF32Func = int (*)(const void*, float, void*); +using KernelBlocksF32Func = int (*)(const void*, float, void*, unsigned int); +using KernelBlockF64Func = int (*)(const void*, double, void*); +using KernelBlocksF64Func = int (*)(const void*, double, void*, unsigned int); + +template +struct Kernel { + std::string_view name; + FuncType func; + + explicit operator bool() const noexcept { + return func != nullptr; + } + + Kernel(const std::string_view name, FuncType func) : name(name), func(func) { + } +}; +} + +#endif //PERNIX_KERNEL_H diff --git a/src/internal/pernix/dispatch/select.h b/src/internal/pernix/dispatch/select.h new file mode 100644 index 0000000..152117a --- /dev/null +++ b/src/internal/pernix/dispatch/select.h @@ -0,0 +1,159 @@ +#ifndef PERNIX_SELECT_H +#define PERNIX_SELECT_H + +#include +#include + +namespace pernix::internal { +Kernel select_compress_block_f32(Backend backend, uint8_t bit_width, uint32_t block_size); + +Kernel select_compress_blocks_f32(Backend backend, uint8_t bit_width, uint32_t block_size); + +Kernel select_compress_block_f64(Backend backend, uint8_t bit_width, uint32_t block_size); + +Kernel select_compress_blocks_f64(Backend backend, uint8_t bit_width, uint32_t block_size); + +Kernel select_decompress_block_f32(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_decompress_blocks_f32(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_decompress_block_f64(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_decompress_blocks_f64(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values); + + +Kernel select_auto_compress_block_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_auto_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_auto_compress_block_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_auto_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_auto_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_auto_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_auto_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_auto_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + + +Kernel select_fallback_compress_block_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_fallback_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_fallback_compress_block_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_fallback_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_fallback_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_fallback_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_fallback_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_fallback_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +#if defined(PERNIX_BUILD_X86_AVX2) + +Kernel select_avx2_compress_block_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_avx2_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_avx2_compress_block_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_avx2_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_avx2_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_avx2_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_avx2_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_avx2_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + +Kernel select_bmi2_compress_block_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_bmi2_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_bmi2_compress_block_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_bmi2_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_bmi2_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_bmi2_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_bmi2_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_bmi2_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + +Kernel select_avx512vbmi_compress_block_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_avx512vbmi_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_avx512vbmi_compress_block_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_avx512vbmi_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_avx512vbmi_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_avx512vbmi_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_avx512vbmi_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_avx512vbmi_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + +Kernel select_neon_compress_block_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_neon_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_neon_compress_block_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_neon_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_neon_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_neon_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_neon_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_neon_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + +Kernel select_sve2_compress_block_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_sve2_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + +Kernel select_sve2_compress_block_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_sve2_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + +Kernel select_sve2_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_sve2_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_sve2_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +Kernel select_sve2_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + +#endif +} + +#endif //PERNIX_SELECT_H diff --git a/include/pernix/fallback/compression.h b/src/internal/pernix/fallback/avx2_compression.h similarity index 73% rename from include/pernix/fallback/compression.h rename to src/internal/pernix/fallback/avx2_compression.h index 5b2780b..c7f03bc 100644 --- a/include/pernix/fallback/compression.h +++ b/src/internal/pernix/fallback/avx2_compression.h @@ -145,9 +145,12 @@ void pack_epi32_fallback(const std::vector& input, uint8_t* __restrict * @param output pointer to the output buffer where compressed bytes will be stored. * @return int status code (0 for success). */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_block_fallback(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { +int compress_block_fallback(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; std::memset(output, 0, BLOCK_SIZE); @@ -173,9 +176,12 @@ int compress_block_fallback(const float_t* __restrict__ input, const float_t sca * @param output pointer to the output buffer where compressed bytes will be stored. * @return int status code (0 for success). */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_block_fallback(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { +int compress_block_fallback(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; std::memset(output, 0, BLOCK_SIZE); @@ -203,9 +209,12 @@ int compress_block_fallback(const double_t* __restrict__ input, const double_t s * @param blocks number of 512-bit blocks to compress. * @return int status code (0 for success). */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_blocks_fallback(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { +int compress_blocks_fallback(const void* __restrict__ input_ptr, float scale, void* __restrict__ output_ptr, uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const float_t* block_input = input; uint8_t* block_output = output; @@ -229,10 +238,13 @@ int compress_blocks_fallback(const float_t* __restrict__ input, const float_t sc * @param blocks number of blocks to compress. * @return int status code (0 for success). */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_blocks_fallback(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { +int compress_blocks_fallback(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, + const unsigned int blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const double_t* block_input = input; uint8_t* block_output = output; @@ -244,63 +256,4 @@ int compress_blocks_fallback(const double_t* __restrict__ input, const double_t return 0; } } // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Compress a single 512-bit block using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - */ -int compress_block_fallback(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single 512-bit block using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - */ -int compress_block_fallback_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress multiple 512-bit blocks using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - */ -int compress_blocks_fallback(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Compress multiple 512-bit blocks using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - */ -int compress_blocks_fallback_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - #endif // PERNIX_FALLBACK_COMPRESSION_H diff --git a/src/internal/pernix/fallback/avx2_decompression.h b/src/internal/pernix/fallback/avx2_decompression.h new file mode 100644 index 0000000..f8f4680 --- /dev/null +++ b/src/internal/pernix/fallback/avx2_decompression.h @@ -0,0 +1,248 @@ +#ifndef PERNIX_FALLBACK_DECOMPRESSION_H +#define PERNIX_FALLBACK_DECOMPRESSION_H + +#include + +#include +#include +#include +#include + +namespace pernix { +namespace internal { +/** +* @brief Dequantize a single int32_t value to float using the provided scale. +* +* @param input input int32_t value to be dequantized. +* @param scale scaling factor used during quantization. +* @return float dequantized float value. +*/ +__always_inline float dequantize_epi32(const int32_t input, const float scale) { + return static_cast(input) * scale; +} + +/** +* @brief Dequantize a single int64_t value to double using the provided scale. +* +* @param input input int64_t value to be dequantized. +* @param scale scaling factor used during quantization. +* @return double_t dequantized double value. +*/ +__always_inline double_t dequantize_epi64(const int64_t input, const double_t scale) { + return static_cast(input) * scale; +} + +/** +* @brief Sign-extend a packed integer value stored in the low bits of a 32-bit word. +* +* @tparam BIT_WIDTH number of significant bits in the encoded value. +* @param value unsigned packed value. +* @return int32_t sign-extended value. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__always_inline auto sign_extend(const uint32_t value) -> int32_t { + if constexpr (BIT_WIDTH == 1) { + return static_cast(value & 1U); + } + + constexpr uint32_t sign_bit = uint32_t{1} << (BIT_WIDTH - 1); + constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1; + const uint32_t masked = value & mask; + return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); +} + +/** +* @brief Unpack bit-packed values from a typed input span into signed 32-bit integers. +* +* @tparam T unsigned integer type used to read the source buffer. +* @tparam BIT_WIDTH bit width per packed value. +* @tparam SIGN_VALUES whether to sign-extend unpacked values. +* @param input pointer to the typed packed input buffer. +* @param bit_offset starting bit offset in the first input word. +* @param elements number of values to unpack. +* @return std::vector unpacked values. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) +__always_inline auto unpack_epi32_fallback_inner(const uint8_t* __restrict__ input, const uint8_t bit_offset, + const std::size_t elements) + -> std::vector { + constexpr uint32_t bits_in_type = sizeof(T) * 8; + constexpr uint32_t bitmask = BIT_WIDTH == bits_in_type + ? std::numeric_limits::max() + : (1U << BIT_WIDTH) - 1U; + + std::vector output(elements); + + std::size_t idx = 0; + uint8_t bits_in_buffer = 8 - bit_offset; + uint64_t buffer = static_cast(input[idx++]) >> bit_offset; + +#pragma GCC unroll 64 + for (uint32_t i = 0; i < elements; i++) { + while (BIT_WIDTH > bits_in_buffer) { + const auto next_value = static_cast(input[idx++]) << bits_in_buffer; + buffer |= next_value; + bits_in_buffer += 8; + } + + const uint32_t raw_value = static_cast(buffer & bitmask); + if constexpr (SIGN_VALUES) { + output[i] = sign_extend(raw_value); + } else { + output[i] = static_cast(raw_value); + } + + buffer >>= BIT_WIDTH; + bits_in_buffer -= BIT_WIDTH; + } + + return output; +} + +/** +* @brief Unpack packed int32_t values from the input buffer using fallback scalar implementation. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @param input pointer to the start of the packed data. +* @param elements number of elements to unpack. +* @return std::vector unpacked int32_t values. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__always_inline auto unpack_epi32_fallback(const uint8_t* __restrict__ input, + const std::size_t elements) -> std::vector { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return unpack_epi32_fallback_inner(input, 0, elements); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return unpack_epi32_fallback_inner(input, 0, elements); + } else { + return unpack_epi32_fallback_inner(input, 0, elements); + } +} +} // namespace internal + +/** +* @brief Decompress a single 512\-bit block using fallback scalar implementation. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @param input pointer to the start of the compressed block. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed float values will be stored. +* @return int status code (0 for success). +*/ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +int decompress_block_fallback(const void* __restrict__ input_ptr, const float_t scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const std::vector block_values = internal::unpack_epi32_fallback( + input, elements_per_block); + +#pragma GCC unroll 512 + for (uint32_t i = 0; i < elements_per_block; i++) { + output[i] = internal::dequantize_epi32(block_values[i], scale); + } + + return 0; +} + +/** +* @brief Decompress a single block to double values using the fallback scalar implementation. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @param input pointer to the start of the compressed block. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed double values will be stored. +* @return int status code (0 for success). +*/ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +int decompress_block_fallback(const void* __restrict__ input_ptr, const double_t scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const std::vector block_values = internal::unpack_epi32_fallback( + input, elements_per_block); + +#pragma GCC unroll 512 + for (uint32_t i = 0; i < elements_per_block; i++) { + output[i] = internal::dequantize_epi64(block_values[i], scale); + } + + return 0; +} + +/** +* @brief Decompress multiple 512\-bit blocks using fallback scalar implementation. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @param input pointer to the start of the compressed data. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed float values will be stored. +* @param blocks number of 512-bit blocks to decompress. +* @return int status code (0 for success). +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +int decompress_blocks_fallback(const void* __restrict__ input_ptr, const float_t scale, + void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const uint8_t* block_input = input; + float_t* block_output = output; + + for (uint32_t block = 0; block < blocks; block++) { + decompress_block_fallback(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; +} + +/** +* @brief Decompress multiple blocks to double values using the fallback scalar implementation. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @param input pointer to the start of the compressed data. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed double values will be stored. +* @param blocks number of blocks to decompress. +* @return int status code (0 for success). +*/ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +int decompress_blocks_fallback(const void* __restrict__ input_ptr, const double_t scale, + void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const uint8_t* block_input = input; + double_t* block_output = output; + + for (uint32_t block = 0; block < blocks; block++) { + decompress_block_fallback(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; +} +} // namespace pernix + +#endif // PERNIX_FALLBACK_DECOMPRESSION_H diff --git a/include/pernix/simd_compat.h b/src/internal/pernix/simd_compat.h similarity index 82% rename from include/pernix/simd_compat.h rename to src/internal/pernix/simd_compat.h index 509eb13..b9dac8e 100644 --- a/include/pernix/simd_compat.h +++ b/src/internal/pernix/simd_compat.h @@ -1,6 +1,7 @@ #ifndef PERNIX_SIMD_COMPAT_H #define PERNIX_SIMD_COMPAT_H +#include #include #include @@ -45,14 +46,4 @@ #endif #endif -#ifndef __always_inline -#if defined(__GNUC__) || defined(__clang__) -#define __always_inline inline __attribute__((always_inline)) -#elif defined(_MSC_VER) -#define __always_inline __forceinline -#else -#define __always_inline inline -#endif -#endif - #endif // PERNIX_SIMD_COMPAT_H diff --git a/include/pernix/x86/avx2/compression.h b/src/internal/pernix/x86/avx2/avx2_compression.h similarity index 67% rename from include/pernix/x86/avx2/compression.h rename to src/internal/pernix/x86/avx2/avx2_compression.h index 357947e..3486f02 100644 --- a/include/pernix/x86/avx2/compression.h +++ b/src/internal/pernix/x86/avx2/avx2_compression.h @@ -1,8 +1,8 @@ #ifndef PERNIX_AVX2_COMPRESSION_H #define PERNIX_AVX2_COMPRESSION_H -#include -#include +#include +#include #include #include @@ -17,16 +17,17 @@ template __always_inline __m256i mm256_clamp_signed_epi32(__m256i input) { constexpr int32_t min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); constexpr int32_t max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), _mm256_set1_epi32(max_value)); + return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), + _mm256_set1_epi32(max_value)); } /** - * @brief Quantize four float values into signed 32-bit integers. - * - * @param input source float lane values. - * @param scale per-lane scale factor. - * @return __m128i quantized values. - */ +* @brief Quantize four float values into signed 32-bit integers. +* +* @param input source float lane values. +* @param scale per-lane scale factor. +* @return __m128i quantized values. +*/ __always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& scale) { const __m128 scaled = _mm_mul_ps(input, scale); // const __m128 rounded = _mm_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); @@ -34,24 +35,24 @@ __always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& } /** - * @brief Quantize two double values into a partially filled 128-bit integer register. - * - * @param input source double lane values. - * @param scale per-lane scale factor. - * @return __m128i quantized values in the low lanes. - */ +* @brief Quantize two double values into a partially filled 128-bit integer register. +* +* @param input source double lane values. +* @param scale per-lane scale factor. +* @return __m128i quantized values in the low lanes. +*/ __always_inline __m128i mm_quantize_pd_epi32(const __m128d& input, const __m128d& scale) { const __m128d scaled = _mm_mul_pd(input, scale); return _mm_cvtpd_epi32(scaled); } /** - * @brief Quantize eight float values into signed 32-bit integers. - * - * @param input source float lane values. - * @param scale per-lane scale factor. - * @return __m256i quantized values. - */ +* @brief Quantize eight float values into signed 32-bit integers. +* +* @param input source float lane values. +* @param scale per-lane scale factor. +* @return __m256i quantized values. +*/ __always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m256& scale) { const __m256 scaled = _mm256_mul_ps(input, scale); // const __m256 rounded = _mm256_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); @@ -59,12 +60,12 @@ __always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m25 } /** - * @brief Quantize four double values into signed 32-bit integers. - * - * @param input source double lane values. - * @param scale per-lane scale factor. - * @return __m128i quantized values in the low lanes. - */ +* @brief Quantize four double values into signed 32-bit integers. +* +* @param input source double lane values. +* @param scale per-lane scale factor. +* @return __m128i quantized values in the low lanes. +*/ __always_inline __m128i mm256_quantize_pd_epi32(const __m256d& input, const __m256d& scale) { const __m256d scaled = _mm256_mul_pd(input, scale); return _mm256_cvtpd_epi32(scaled); @@ -72,12 +73,12 @@ __always_inline __m128i mm256_quantize_pd_epi32(const __m256d& input, const __m2 #ifndef PERNIX_USE_SIMDE /** - * @brief Emulate per-16-bit left shifts on AVX2. - * - * @param a source values. - * @param count per-lane shift amounts. - * @return __m128i shifted values. - */ +* @brief Emulate per-16-bit left shifts on AVX2. +* +* @param a source values. +* @param count per-lane shift amounts. +* @return __m128i shifted values. +*/ __always_inline static __m128i _mm_sllv_epi16(const __m128i a, const __m128i count) { const __m128i mask = _mm_set1_epi32(0xffff0000); const __m128i low_half = _mm_sllv_epi32(a, _mm_andnot_si128(mask, count)); @@ -86,8 +87,8 @@ __always_inline static __m128i _mm_sllv_epi16(const __m128i a, const __m128i cou } /** - * @brief Emulate per-16-bit right shifts on AVX2. - */ +* @brief Emulate per-16-bit right shifts on AVX2. +*/ __always_inline static __m128i _mm_srlv_epi16(const __m128i a, const __m128i count) { const __m128i mask = _mm_set1_epi32(0x0000ffff); const __m128i low_half = _mm_srlv_epi32(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); @@ -96,8 +97,8 @@ __always_inline static __m128i _mm_srlv_epi16(const __m128i a, const __m128i cou } /** - * @brief Emulate per-16-bit left shifts on 256-bit AVX2 vectors. - */ +* @brief Emulate per-16-bit left shifts on 256-bit AVX2 vectors. +*/ __always_inline static __m256i _mm256_sllv_epi16(const __m256i a, const __m256i count) { const __m256i mask = _mm256_set1_epi32(0xffff0000); const __m256i low_half = _mm256_sllv_epi32(a, _mm256_andnot_si256(mask, count)); @@ -106,8 +107,8 @@ __always_inline static __m256i _mm256_sllv_epi16(const __m256i a, const __m256i } /** - * @brief Emulate per-16-bit right shifts on 256-bit AVX2 vectors. - */ +* @brief Emulate per-16-bit right shifts on 256-bit AVX2 vectors. +*/ __always_inline static __m256i _mm256_srlv_epi16(const __m256i a, const __m256i count) { const __m256i mask = _mm256_set1_epi32(0x0000ffff); const __m256i low_half = _mm256_srlv_epi32(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); @@ -116,22 +117,22 @@ __always_inline static __m256i _mm256_srlv_epi16(const __m256i a, const __m256i } /** - * @brief Blend 8-bit lanes by expanding a scalar mask value. - */ +* @brief Blend 8-bit lanes by expanding a scalar mask value. +*/ __always_inline static __m128i mm_blend_epi8(const __m128i X, const __m128i Y, const int8_t M) { return _mm_blendv_epi8(X, Y, _mm_set1_epi8(M)); } /** - * @brief Blend 8-bit lanes in 256-bit vectors by expanding a scalar mask value. - */ +* @brief Blend 8-bit lanes in 256-bit vectors by expanding a scalar mask value. +*/ __always_inline static __m256i mm256_blend_epi8(const __m256i X, const __m256i Y, const int8_t M) { return _mm256_blendv_epi8(X, Y, _mm256_set1_epi8(M)); } /** - * @brief Emulate per-byte left shifts on 128-bit vectors. - */ +* @brief Emulate per-byte left shifts on 128-bit vectors. +*/ __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { const __m128i mask = _mm_set1_epi16(0xff00); const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); @@ -140,8 +141,8 @@ __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i coun } /** - * @brief Emulate per-byte right shifts on 128-bit vectors. - */ +* @brief Emulate per-byte right shifts on 128-bit vectors. +*/ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i count) { const __m128i mask = _mm_set1_epi16(0x00ff); const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); @@ -150,8 +151,8 @@ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i coun } /** - * @brief Emulate per-byte left shifts on 256-bit vectors. - */ +* @brief Emulate per-byte left shifts on 256-bit vectors. +*/ __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { const __m256i mask = _mm256_set1_epi16(0xff00); const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); @@ -160,8 +161,8 @@ __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i c } /** - * @brief Emulate per-byte right shifts on 256-bit vectors. - */ +* @brief Emulate per-byte right shifts on 256-bit vectors. +*/ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i count) { const __m256i mask = _mm256_set1_epi16(0x00ff); const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); @@ -171,8 +172,8 @@ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i c #endif /** - * @brief Pack four 32-bit values for bit widths 1 through 3. - */ +* @brief Pack four 32-bit values for bit widths 1 through 3. +*/ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) __always_inline auto mm_pack_epi32_avx2_1to3(__m128i& input) -> __m128i { @@ -182,15 +183,16 @@ __always_inline auto mm_pack_epi32_avx2_1to3(__m128i& input) -> __m128i { alignas(16) uint32_t lanes[4]; _mm_storeu_si128(reinterpret_cast<__m128i*>(lanes), masked); - const uint32_t packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ((lanes[2] & bitmask) << (2 * BIT_WIDTH)) | + const uint32_t packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ( + (lanes[2] & bitmask) << (2 * BIT_WIDTH)) | ((lanes[3] & bitmask) << (3 * BIT_WIDTH)); return _mm_cvtsi32_si128(static_cast(packed)); } /** - * @brief Pack eight 32-bit values for bit widths 1 through 3. - */ +* @brief Pack eight 32-bit values for bit widths 1 through 3. +*/ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) __always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { @@ -198,7 +200,8 @@ __always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(static_cast(bitmask))); - const __m256i shifts = _mm256_setr_epi32(0 * BIT_WIDTH, 1 * BIT_WIDTH, 2 * BIT_WIDTH, 3 * BIT_WIDTH, 4 * BIT_WIDTH, 5 * BIT_WIDTH, + const __m256i shifts = _mm256_setr_epi32(0 * BIT_WIDTH, 1 * BIT_WIDTH, 2 * BIT_WIDTH, 3 * BIT_WIDTH, + 4 * BIT_WIDTH, 5 * BIT_WIDTH, 6 * BIT_WIDTH, 7 * BIT_WIDTH); const __m256i shifted = _mm256_sllv_epi32(masked, shifts); @@ -220,15 +223,17 @@ __always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i& input) { const __m256i combined = _mm256_or_si256(packed8, _mm256_srli_epi16(packed8, 4)); - const __m256i shuffled = _mm256_shuffle_epi8(combined, _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, - 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i shuffled = _mm256_shuffle_epi8(combined, _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 2, + 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1)); return shuffled; } /** - * @brief Pack four 32-bit values for bit widths 9 through 16. - */ +* @brief Pack four 32-bit values for bit widths 9 through 16. +*/ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { @@ -260,8 +265,8 @@ __always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { } /** - * @brief Pack eight 32-bit values for bit widths 9 through 16. - */ +* @brief Pack eight 32-bit values for bit widths 9 through 16. +*/ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i& input) -> __m256i { @@ -312,8 +317,8 @@ __always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i } /** - * @brief Pack eight 32-bit values for bit widths 17 through 24. - */ +* @brief Pack eight 32-bit values for bit widths 17 through 24. +*/ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m256i { @@ -346,8 +351,8 @@ __always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m25 } /** - * @brief Pack aligned 8-bit or 16-bit values from four 32-bit lanes. - */ +* @brief Pack aligned 8-bit or 16-bit values from four 32-bit lanes. +*/ template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { @@ -359,8 +364,8 @@ auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { } /** - * @brief Dispatch to the appropriate 128-bit AVX2 packer for the selected bit width. - */ +* @brief Dispatch to the appropriate 128-bit AVX2 packer for the selected bit width. +*/ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 16) auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { @@ -377,23 +382,25 @@ auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { } /** - * @brief Pack aligned 8-bit or 16-bit values from eight 32-bit lanes. - */ +* @brief Pack aligned 8-bit or 16-bit values from eight 32-bit lanes. +*/ template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) __m256i mm256_pack_aligned_epi32_avx2(const __m256i& input) { if constexpr (BIT_WIDTH == 8) { - const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); - const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); + const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), + _mm256_extracti128_si256(input, 1)); + const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); return _mm256_castsi128_si256(packed8); } else { - return _mm256_castsi128_si256(_mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1))); + return _mm256_castsi128_si256( + _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1))); } } /** - * @brief Dispatch to the appropriate 256-bit AVX2 packer for the selected bit width. - */ +* @brief Dispatch to the appropriate 256-bit AVX2 packer for the selected bit width. +*/ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __m256i mm256_pack_epi32_avx2(const __m256i& input) { @@ -417,21 +424,25 @@ __m256i mm256_pack_epi32_avx2(const __m256i& input) { } // namespace internal /** - * @brief Compress a single block of float using AVX2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -template +* @brief Compress a single block of float using AVX2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). +* +* @param input pointer to the start of the input float values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX2 support. +*/ +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_avx2(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { +int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const float_t scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; @@ -444,7 +455,7 @@ int mm256_compress_block_avx2(const float_t* __restrict__ input, const float_t s const __m256 source = _mm256_loadu_ps(input); const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); - const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); std::memcpy(output, &packed, BIT_WIDTH); input += 8; @@ -456,7 +467,8 @@ int mm256_compress_block_avx2(const float_t* __restrict__ input, const float_t s #pragma GCC unroll 8 for (uint32_t i = 0; i < remaining; i++) { block_values[i] = - static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], scale))); + static_cast(internal::clamp_signed_quantized( + internal::quantize_ps_epi32(input[i], scale))); } internal::pack_epi32_fallback(block_values, output); @@ -466,21 +478,25 @@ int mm256_compress_block_avx2(const float_t* __restrict__ input, const float_t s } /** - * @brief Compress a single block of double using AVX2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -template +* @brief Compress a single block of double using AVX2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). +* +* @param input pointer to the start of the input double values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX2 support. +*/ +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_avx2(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { +int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const double_t scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; @@ -496,7 +512,8 @@ int mm256_compress_block_avx2(const double_t* __restrict__ input, const double_t const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); __m256i combined = _mm256_castsi128_si256(quantized1); combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_avx2(internal::mm256_clamp_signed_epi32(combined)); + const __m256i packed = internal::mm256_pack_epi32_avx2( + internal::mm256_clamp_signed_epi32(combined)); // _mm_storeu_si128(reinterpret_cast<__m128i*>(output), _mm256_castsi256_si128(packed)); std::memcpy(output, &packed, BIT_WIDTH); input += 8; @@ -508,7 +525,8 @@ int mm256_compress_block_avx2(const double_t* __restrict__ input, const double_t #pragma GCC unroll 8 for (uint32_t i = 0; i < remaining; i++) { block_values[i] = - static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], scale))); + static_cast(internal::clamp_signed_quantized( + internal::quantize_pd_epi64(input[i], scale))); } internal::pack_epi32_fallback(block_values, output); @@ -518,23 +536,27 @@ int mm256_compress_block_avx2(const double_t* __restrict__ input, const double_t } /** - * @brief Compress multiple blocks using AVX2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -template +* @brief Compress multiple blocks using AVX2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). +* +* @param input pointer to the start of the input float values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @param blocks number of blocks to compress. +* @return int status code (0 for success). +* +* @note This function requires AVX2 support. +*/ +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_avx2(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, +int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const float_t scale, + void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const float_t* block_input = input; uint8_t* block_output = output; @@ -548,23 +570,27 @@ int mm256_compress_blocks_avx2(const float_t* __restrict__ input, const float_t } /** - * @brief Compress multiple blocks using AVX2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -template +* @brief Compress multiple blocks using AVX2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). +* +* @param input pointer to the start of the input double values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @param blocks number of blocks to compress. +* @return int status code (0 for success). +* +* @note This function requires AVX2 support. +*/ +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_avx2(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, +int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const double_t scale, + void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const double_t* block_input = input; uint8_t* block_output = output; @@ -578,70 +604,4 @@ int mm256_compress_blocks_avx2(const double_t* __restrict__ input, const double_ } } // namespace pernix -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Compress a single 512-bit block using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_compress_block_avx2(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single 512-bit block using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_compress_block_f64_avx2(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress multiple 512-bit blocks using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_blocks_avx2(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Compress multiple 512-bit blocks using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_blocks_f64_avx2(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - #endif // PERNIX_AVX2_COMPRESSION_H diff --git a/include/pernix/x86/avx2/decompression.h b/src/internal/pernix/x86/avx2/avx2_decompression.h similarity index 79% rename from include/pernix/x86/avx2/decompression.h rename to src/internal/pernix/x86/avx2/avx2_decompression.h index 93cbce5..7c530d8 100644 --- a/include/pernix/x86/avx2/decompression.h +++ b/src/internal/pernix/x86/avx2/avx2_decompression.h @@ -1,8 +1,8 @@ #ifndef PERNIX_AVX2_DECOMPRESSION_H #define PERNIX_AVX2_DECOMPRESSION_H -#include -#include +#include +#include #include #include @@ -190,9 +190,12 @@ __m256i mm256_unpack_epi32_avx2(const uint8_t* __restrict__ input) { * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_avx2(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; @@ -230,9 +233,12 @@ int mm256_decompress_block_avx2(const uint8_t* __restrict__ input, const float_t * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_avx2(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; @@ -276,10 +282,13 @@ int mm256_decompress_block_avx2(const uint8_t* __restrict__ input, const double_ * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_avx2(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, +int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const uint8_t* block_input = input; float_t* block_output = output; @@ -306,10 +315,13 @@ int mm256_decompress_blocks_avx2(const uint8_t* __restrict__ input, const float_ * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_avx2(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, +int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const uint8_t* block_input = input; double_t* block_output = output; @@ -323,70 +335,4 @@ int mm256_decompress_blocks_avx2(const uint8_t* __restrict__ input, const double } } // namespace pernix -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Decompress a single 512-bit block to float using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_decompress_block_avx2(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single 512-bit block to double using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_decompress_block_f64_avx2(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -/** - * @brief Decompress multiple 512-bit blocks using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_decompress_blocks_avx2(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Decompress multiple 512-bit blocks to double using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_decompress_blocks_f64_avx2(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - #endif // PERNIX_AVX2_DECOMPRESSION_H diff --git a/include/pernix/x86/avx2/tables.h b/src/internal/pernix/x86/avx2/avx2_tables.h similarity index 68% rename from include/pernix/x86/avx2/tables.h rename to src/internal/pernix/x86/avx2/avx2_tables.h index f4f374b..e21af27 100644 --- a/include/pernix/x86/avx2/tables.h +++ b/src/internal/pernix/x86/avx2/avx2_tables.h @@ -7,10 +7,10 @@ #include namespace pernix::internal { -template <__uint8_t BIT_WIDTH, typename T> - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && (std::is_same_v || std::is_same_v)) -struct pack_tables_avx2_16 { - alignas(64) inline static constexpr std::array permute1 = [] { + template<__uint8_t BIT_WIDTH, typename T> + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && (std::is_same_v || std::is_same_v)) + struct pack_tables_avx2_16 { + alignas(64) inline static constexpr std::array permute1 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { return std::array{ @@ -98,10 +98,10 @@ struct pack_tables_avx2_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute2 = [] { + alignas(64) inline static constexpr std::array permute2 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { return std::array{ @@ -189,10 +189,10 @@ struct pack_tables_avx2_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute3 = [] { + alignas(64) inline static constexpr std::array permute3 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { return std::array{ @@ -244,10 +244,10 @@ struct pack_tables_avx2_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift1 = [] { + alignas(64) inline static constexpr std::array shift1 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { return std::array{ @@ -286,10 +286,10 @@ struct pack_tables_avx2_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift2 = [] { + alignas(64) inline static constexpr std::array shift2 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { return std::array{ @@ -328,10 +328,10 @@ struct pack_tables_avx2_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift3 = [] { + alignas(64) inline static constexpr std::array shift3 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { return std::array{ @@ -355,71 +355,71 @@ struct pack_tables_avx2_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" __always_inline static T get_permute1() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute1.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute1.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute1.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute1.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_permute2() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute2.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute2.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute2.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute2.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_permute3() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute3.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute3.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute3.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute3.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_shift1() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift1.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift1.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift1.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift1.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_shift2() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift2.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift2.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift2.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift2.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_shift3() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift3.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift3.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift3.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift3.data())); + } + return T{}; } - return T{}; - } #pragma GCC diagnostic pop -}; + }; -template <__uint8_t BIT_WIDTH, typename T> - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) -struct pack_tables_avx2_24 { - alignas(64) inline static constexpr std::array permute1 = [] { + template<__uint8_t BIT_WIDTH, typename T> + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) + struct pack_tables_avx2_24 { + alignas(64) inline static constexpr std::array permute1 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -455,10 +455,10 @@ struct pack_tables_avx2_24 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute2 = [] { + alignas(64) inline static constexpr std::array permute2 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -494,10 +494,10 @@ struct pack_tables_avx2_24 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute3 = [] { + alignas(64) inline static constexpr std::array permute3 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -529,10 +529,10 @@ struct pack_tables_avx2_24 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift1 = [] { + alignas(64) inline static constexpr std::array shift1 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -568,10 +568,10 @@ struct pack_tables_avx2_24 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift2 = [] { + alignas(64) inline static constexpr std::array shift2 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -607,10 +607,10 @@ struct pack_tables_avx2_24 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift3 = [] { + alignas(64) inline static constexpr std::array shift3 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -642,71 +642,71 @@ struct pack_tables_avx2_24 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" __always_inline static T get_permute1() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute1.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute1.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute1.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute1.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_permute2() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute2.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute2.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute2.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute2.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_permute3() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute3.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute3.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute3.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute3.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_shift1() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift1.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift1.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift1.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift1.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_shift2() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift2.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift2.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift2.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift2.data())); + } + return T{}; } - return T{}; - } __always_inline static T get_shift3() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift3.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift3.data())); + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift3.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift3.data())); + } + return T{}; } - return T{}; - } #pragma GCC diagnostic pop -}; + }; -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) -struct unpack_tables_avx2 { - alignas(32) inline static constexpr std::array permute = [] { + template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) + struct unpack_tables_avx2 { + alignas(32) inline static constexpr std::array permute = [] { // clang-format off if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return std::array{0, -1, -1, -1, 0, 1, -1, -1}; @@ -715,71 +715,72 @@ struct unpack_tables_avx2 { } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { return std::array{0, 1, 2, -1, 2, 3, 4, 5}; } - // clang-format on - }(); - - alignas(32) inline static constexpr std::array shuffle = [] { - std::array shuffles{}; - shuffles.fill(-1); - constexpr std::size_t rebase_second_half = 4 * ((BIT_WIDTH - 1) / 8); - - for (std::size_t lane = 0; lane < 2; ++lane) { - for (std::size_t i = 0; i < 4; ++i) { - const std::size_t value_index = lane * 4 + i; - - const std::size_t bit_start = value_index * BIT_WIDTH; - const std::size_t byte_start = bit_start / 8; - const std::size_t bit_offset = bit_start % 8; - const std::size_t byte_count = (bit_offset + BIT_WIDTH + 7) / 8; - - const std::size_t rebase = (lane == 0) ? 0 : rebase_second_half; - const std::size_t rel_byte_start = byte_start - rebase; - - const std::size_t dst = (lane * 4 + i) * 4; - for (std::size_t k = 0; k < byte_count; ++k) { - shuffles[dst + k] = static_cast(rel_byte_start + k); + // clang-format on + }(); + + alignas(32) inline static constexpr std::array shuffle = [] { + std::array shuffles{}; + shuffles.fill(-1); + constexpr std::size_t rebase_second_half = 4 * ((BIT_WIDTH - 1) / 8); + + for (std::size_t lane = 0; lane < 2; ++lane) { + for (std::size_t i = 0; i < 4; ++i) { + const std::size_t value_index = lane * 4 + i; + + const std::size_t bit_start = value_index * BIT_WIDTH; + const std::size_t byte_start = bit_start / 8; + const std::size_t bit_offset = bit_start % 8; + const std::size_t byte_count = (bit_offset + BIT_WIDTH + 7) / 8; + + const std::size_t rebase = (lane == 0) ? 0 : rebase_second_half; + const std::size_t rel_byte_start = byte_start - rebase; + + const std::size_t dst = (lane * 4 + i) * 4; + for (std::size_t k = 0; k < byte_count; ++k) { + shuffles[dst + k] = static_cast(rel_byte_start + k); + } } } - } - return shuffles; - }(); + return shuffles; + }(); - alignas(64) inline static constexpr std::array shift = [] { - std::array shifts{}; + alignas(64) inline static constexpr std::array shift = [] { + std::array shifts{}; - for (std::size_t lane = 0; lane < 8; ++lane) { - const int bit_offset = lane * BIT_WIDTH; - const int bit_in_byte = bit_offset % 8; - const int left_shift = 32 - BIT_WIDTH - bit_in_byte; - shifts[lane] = left_shift; - } + for (std::size_t lane = 0; lane < 8; ++lane) { + const int bit_offset = lane * BIT_WIDTH; + const int bit_in_byte = bit_offset % 8; + const int left_shift = 32 - BIT_WIDTH - bit_in_byte; + shifts[lane] = left_shift; + } - return shifts; - }(); + return shifts; + }(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" - __always_inline static __m256i get_permute() { return _mm256_load_si256(reinterpret_cast(permute.data())); } + __always_inline static __m256i get_permute() { + return _mm256_load_si256(reinterpret_cast(permute.data())); + } __always_inline static T get_shuffle() { - if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shuffle.data())); - } else { - return _mm256_load_si256(reinterpret_cast(shuffle.data())); + if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shuffle.data())); + } else { + return _mm256_load_si256(reinterpret_cast(shuffle.data())); + } } - } __always_inline static T get_shift() { - if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift.data())); - } else { - return _mm256_load_si256(reinterpret_cast(shift.data())); + if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift.data())); + } else { + return _mm256_load_si256(reinterpret_cast(shift.data())); + } } - } #pragma GCC diagnostic pop -}; - -} // namespace pernix::internal + }; +} // namespace pernix::internal #endif // PERNIX_AVX2_TABLES_H diff --git a/include/pernix/x86/avx512vbmi/compression.h b/src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h similarity index 88% rename from include/pernix/x86/avx512vbmi/compression.h rename to src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h index 9621e12..e19051e 100644 --- a/include/pernix/x86/avx512vbmi/compression.h +++ b/src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h @@ -2,10 +2,10 @@ #define PERNIX_AVX512VBMI_COMPRESSION_H #include -#include -#include -#include +#include #include +#include +#include #include @@ -99,7 +99,7 @@ static __always_inline __m256i make_m256i_from_4x64(const __m128i a, const __m12 return x; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { @@ -181,7 +181,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_compress_block_avx512vbmi_9to16(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { @@ -251,7 +251,7 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const float_t* __restr return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_compress_block_avx512vbmi_17to24(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { @@ -299,7 +299,7 @@ __always_inline int mm512_compress_block_avx512vbmi_17to24(const float_t* __rest return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_compress_block_avx512vbmi_1to8(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { @@ -414,7 +414,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const double_t* __restr return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_compress_block_avx512vbmi_9to16(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { @@ -496,7 +496,7 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const double_t* __rest return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_compress_block_avx512vbmi_17to24(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { @@ -560,9 +560,12 @@ __always_inline int mm512_compress_block_avx512vbmi_17to24(const double_t* __res * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_block_avx512vbmi(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { +int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + std::memset(output, 0, BLOCK_SIZE); if constexpr (BIT_WIDTH <= 8) { @@ -585,9 +588,12 @@ int mm512_compress_block_avx512vbmi(const float_t* __restrict__ input, const flo * * @note This overload is declared for parity with the float path. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_block_avx512vbmi(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { +int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + std::memset(output, 0, BLOCK_SIZE); if constexpr (BIT_WIDTH <= 8) { @@ -612,10 +618,13 @@ int mm512_compress_block_avx512vbmi(const double_t* __restrict__ input, const do * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_blocks_avx512vbmi(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, +int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const float_t* block_input = input; uint8_t* block_output = output; @@ -638,10 +647,13 @@ int mm512_compress_blocks_avx512vbmi(const float_t* __restrict__ input, const fl * @param blocks number of blocks to compress. * @return int status code. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_blocks_avx512vbmi(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, +int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const double_t* block_input = input; uint8_t* block_output = output; @@ -655,71 +667,4 @@ int mm512_compress_blocks_avx512vbmi(const double_t* __restrict__ input, const d } } // namespace pernix -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_compress_block_avx512vbmi(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_compress_block_f64_avx512vbmi(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, - uint8_t* __restrict__ output); - -/** - * @brief Compress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_compress_blocks_avx512vbmi(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Compress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_compress_blocks_f64_avx512vbmi(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, - uint8_t* __restrict__ output, uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - #endif // PERNIX_AVX512VBMI_COMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/decompression.h b/src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h similarity index 87% rename from include/pernix/x86/avx512vbmi/decompression.h rename to src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h index 08abc35..d0f98f0 100644 --- a/include/pernix/x86/avx512vbmi/decompression.h +++ b/src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h @@ -2,7 +2,7 @@ #define PERNIX_AVX512VBMI_DECOMPRESSION_H #include -#include +#include #include #include #include @@ -26,7 +26,7 @@ __always_inline __m512d mm512_dequantize_epi64(const __m512i& input, const __m51 return _mm512_mul_pd(converted, scale); } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { @@ -110,7 +110,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __rest return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { @@ -228,7 +228,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __rest return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { @@ -301,7 +301,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __res return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { @@ -385,7 +385,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __res return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { @@ -437,7 +437,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __re return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { @@ -511,10 +511,13 @@ __always_inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __re * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const float_t scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return internal::mm512_decompress_block_avx512vbmi_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { @@ -535,10 +538,13 @@ __always_inline int mm512_decompress_block_avx512vbmi(const uint8_t* __restrict_ * @param output pointer to the output buffer where decompressed double values will be stored. * @return int status code. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const double_t scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return internal::mm512_decompress_block_avx512vbmi_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { @@ -563,10 +569,13 @@ __always_inline int mm512_decompress_block_avx512vbmi(const uint8_t* __restrict_ * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, +int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const uint8_t* block_input = input; float_t* block_output = output; @@ -590,10 +599,13 @@ int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const * @param blocks number of blocks to decompress. * @return int status code. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, +int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const uint8_t* block_input = input; double_t* block_output = output; @@ -606,70 +618,4 @@ int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const } } // namespace pernix -namespace pernix { -#ifdef __cplusplus -extern "C" { -#endif -/** - * @brief Decompress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_decompress_block_avx512vbmi(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_decompress_block_f64_avx512vbmi(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output); - -/** - * @brief Decompress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_decompress_blocks_avx512vbmi(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Decompress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_decompress_blocks_f64_avx512vbmi(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output, uint32_t blocks); - -#ifdef __cplusplus -} -#endif -} // namespace pernix - #endif // PERNIX_AVX512VBMI_DECOMPRESSION_H diff --git a/src/internal/pernix/x86/avx512vbmi/compat.h b/src/internal/pernix/x86/avx512vbmi/compat.h new file mode 100644 index 0000000..6918d66 --- /dev/null +++ b/src/internal/pernix/x86/avx512vbmi/compat.h @@ -0,0 +1,387 @@ +#ifndef PERNIX_AVX512_COMPAT_H +#define PERNIX_AVX512_COMPAT_H + +#include +#include +#include +#include + +namespace pernix::internal { + static __always_inline __mmask8 element_mask8(const uint32_t e) { + return static_cast<__mmask8>(e >= 8 ? 0xFFu : ((1u << e) - 1u)); + } + + static __always_inline __mmask16 element_mask16(const uint32_t e) { + return static_cast<__mmask16>(e >= 16 ? 0xFFFFu : ((1u << e) - 1u)); + } + + static __always_inline __mmask32 element_mask32(const uint32_t e) { + return e >= 32 ? 0xFFFFFFFFu : (1u << e) - 1u; + } + + static __always_inline __mmask64 element_mask64(const uint32_t e) { + return e >= 64 ? 0xFFFFFFFFFFFFFFFFull : (1ull << e) - 1ull; + } + + static __always_inline __m512i mm512_loadu_elements_epi64(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m512i a = _mm512_setzero_si512(); + std::memcpy(&a, mem_addr, e * sizeof(int64_t)); + return a; +#else + return _mm512_maskz_loadu_epi64(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m256i mm256_loadu_elements_epi64(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m256i a = _mm256_setzero_si256(); + std::memcpy(&a, mem_addr, e * sizeof(int64_t)); + return a; +#else + return _mm256_maskz_loadu_epi64(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m128i mm_loadu_elements_epi64(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m128i a = _mm_setzero_si128(); + std::memcpy(&a, mem_addr, e * sizeof(int64_t)); + return a; +#else + return _mm_maskz_loadu_epi64(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m512i mm512_loadu_elements_epi32(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m512i a = _mm512_setzero_si512(); + std::memcpy(&a, mem_addr, e * sizeof(int32_t)); + return a; +#else + return _mm512_maskz_loadu_epi32(element_mask16(e), mem_addr); +#endif + } + + static __always_inline __m256i mm256_loadu_elements_epi32(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m256i a = _mm256_setzero_si256(); + std::memcpy(&a, mem_addr, e * sizeof(int32_t)); + return a; +#else + return _mm256_maskz_loadu_epi32(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m128i mm_loadu_elements_epi32(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m128i a = _mm_setzero_si128(); + std::memcpy(&a, mem_addr, e * sizeof(int32_t)); + return a; +#else + return _mm_maskz_loadu_epi32(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m512i mm512_loadu_elements_epi16(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m512i a = _mm512_setzero_si512(); + std::memcpy(&a, mem_addr, e * sizeof(int16_t)); + return a; +#else + return _mm512_maskz_loadu_epi16(element_mask32(e), mem_addr); +#endif + } + + static __always_inline __m256i mm256_loadu_elements_epi16(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m256i a = _mm256_setzero_si256(); + std::memcpy(&a, mem_addr, e * sizeof(int16_t)); + return a; +#else + return _mm256_maskz_loadu_epi16(element_mask16(e), mem_addr); +#endif + } + + static __always_inline __m128i mm_loadu_elements_epi16(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m128i a = _mm_setzero_si128(); + std::memcpy(&a, mem_addr, e * sizeof(int16_t)); + return a; +#else + return _mm_maskz_loadu_epi16(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m512i mm512_loadu_elements_epi8(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m512i a = _mm512_setzero_si512(); + std::memcpy(&a, mem_addr, e * sizeof(int8_t)); + return a; +#else + return _mm512_maskz_loadu_epi8(element_mask64(e), mem_addr); +#endif + } + + static __always_inline __m256i mm256_loadu_elements_epi8(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m256i a = _mm256_setzero_si256(); + std::memcpy(&a, mem_addr, e * sizeof(int8_t)); + return a; +#else + return _mm256_maskz_loadu_epi8(element_mask32(e), mem_addr); +#endif + } + + static __always_inline __m128i mm_loadu_elements_epi8(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m128i a = _mm_setzero_si128(); + std::memcpy(&a, mem_addr, e * sizeof(int8_t)); + return a; +#else + return _mm_maskz_loadu_epi8(element_mask16(e), mem_addr); +#endif + } + + static __always_inline void mm512_storeu_elements_epi64(void *mem_addr, const uint32_t e, const __m512i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(64) uint8_t bytes[64]; + _mm512_storeu_si512(bytes, a); + std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); +#else + _mm512_mask_storeu_epi64(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm256_storeu_elements_epi64(void *mem_addr, const uint32_t e, const __m256i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(32) uint8_t bytes[32]; + _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); +#else + _mm256_mask_storeu_epi64(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm_storeu_elements_epi64(void *mem_addr, const uint32_t e, const __m128i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(16) uint8_t bytes[16]; + _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); +#else + _mm_mask_storeu_epi64(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm512_storeu_elements_epi32(void *mem_addr, const uint32_t e, const __m512i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(64) uint8_t bytes[64]; + _mm512_storeu_si512(bytes, a); + std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); +#else + _mm512_mask_storeu_epi32(mem_addr, element_mask16(e), a); +#endif + } + + static __always_inline void mm256_storeu_elements_epi32(void *mem_addr, const uint32_t e, const __m256i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(32) uint8_t bytes[32]; + _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); +#else + _mm256_mask_storeu_epi32(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm_storeu_elements_epi32(void *mem_addr, const uint32_t e, const __m128i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(16) uint8_t bytes[16]; + _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); +#else + _mm_mask_storeu_epi32(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm512_storeu_elements_epi16(void *mem_addr, const uint32_t e, const __m512i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(64) uint8_t bytes[64]; + _mm512_storeu_si512(bytes, a); + std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); +#else + _mm512_mask_storeu_epi16(mem_addr, element_mask32(e), a); +#endif + } + + static __always_inline void mm256_storeu_elements_epi16(void *mem_addr, const uint32_t e, const __m256i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(32) uint8_t bytes[32]; + _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); +#else + _mm256_mask_storeu_epi16(mem_addr, element_mask16(e), a); +#endif + } + + static __always_inline void mm_storeu_elements_epi16(void *mem_addr, const uint32_t e, const __m128i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(16) uint8_t bytes[16]; + _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); +#else + _mm_mask_storeu_epi16(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm512_storeu_elements_epi8(void *mem_addr, const uint32_t e, const __m512i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(64) uint8_t bytes[64]; + _mm512_storeu_si512(bytes, a); + std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); +#else + _mm512_mask_storeu_epi8(mem_addr, element_mask64(e), a); +#endif + } + + static __always_inline void mm256_storeu_elements_epi8(void *mem_addr, const uint32_t e, const __m256i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(32) uint8_t bytes[32]; + _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); +#else + _mm256_mask_storeu_epi8(mem_addr, element_mask32(e), a); +#endif + } + + static __always_inline void mm_storeu_elements_epi8(void *mem_addr, const uint32_t e, const __m128i a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(16) uint8_t bytes[16]; + _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); +#else + _mm_mask_storeu_epi8(mem_addr, element_mask16(e), a); +#endif + } + + static __always_inline __m512 mm512_loadu_elements_ps(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m512 a = _mm512_setzero_ps(); + std::memcpy(&a, mem_addr, e * sizeof(float_t)); + return a; +#else + return _mm512_maskz_loadu_ps(element_mask16(e), mem_addr); +#endif + } + + static __always_inline __m256 mm256_loadu_elements_ps(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m256 a = _mm256_setzero_ps(); + std::memcpy(&a, mem_addr, e * sizeof(float_t)); + return a; +#else + return _mm256_maskz_loadu_ps(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m128 mm_loadu_elements_ps(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m128 a = _mm_setzero_ps(); + std::memcpy(&a, mem_addr, e * sizeof(float_t)); + return a; +#else + return _mm_maskz_loadu_ps(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m512d mm512_loadu_elements_pd(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m512d a = _mm512_setzero_pd(); + std::memcpy(&a, mem_addr, e * sizeof(double_t)); + return a; +#else + return _mm512_maskz_loadu_pd(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m256d mm256_loadu_elements_pd(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m256d a = _mm256_setzero_pd(); + std::memcpy(&a, mem_addr, e * sizeof(double_t)); + return a; +#else + return _mm256_maskz_loadu_pd(element_mask8(e), mem_addr); +#endif + } + + static __always_inline __m128d mm_loadu_elements_pd(const uint32_t e, const void *mem_addr) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + __m128d a = _mm_setzero_pd(); + std::memcpy(&a, mem_addr, e * sizeof(double_t)); + return a; +#else + return _mm_maskz_loadu_pd(element_mask8(e), mem_addr); +#endif + } + + static __always_inline void mm512_storeu_elements_ps(void *mem_addr, const uint32_t e, const __m512 a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(64) float_t values[16]; + _mm512_storeu_ps(values, a); + std::memcpy(mem_addr, values, e * sizeof(float_t)); +#else + _mm512_mask_storeu_ps(mem_addr, element_mask16(e), a); +#endif + } + + static __always_inline void mm256_storeu_elements_ps(void *mem_addr, const uint32_t e, const __m256 a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(32) float_t values[8]; + _mm256_storeu_ps(values, a); + std::memcpy(mem_addr, values, e * sizeof(float_t)); +#else + _mm256_mask_storeu_ps(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm_storeu_elements_ps(void *mem_addr, const uint32_t e, const __m128 a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(16) float_t values[4]; + _mm_storeu_ps(values, a); + std::memcpy(mem_addr, values, e * sizeof(float_t)); +#else + _mm_mask_storeu_ps(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm512_storeu_elements_pd(void *mem_addr, const uint32_t e, const __m512d a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(64) double_t values[8]; + _mm512_storeu_pd(values, a); + std::memcpy(mem_addr, values, e * sizeof(double_t)); +#else + _mm512_mask_storeu_pd(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm256_storeu_elements_pd(void *mem_addr, const uint32_t e, const __m256d a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(32) double_t values[4]; + _mm256_storeu_pd(values, a); + std::memcpy(mem_addr, values, e * sizeof(double_t)); +#else + _mm256_mask_storeu_pd(mem_addr, element_mask8(e), a); +#endif + } + + static __always_inline void mm_storeu_elements_pd(void *mem_addr, const uint32_t e, const __m128d a) { +#if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) + alignas(16) double_t values[2]; + _mm_storeu_pd(values, a); + std::memcpy(mem_addr, values, e * sizeof(double_t)); +#else + _mm_mask_storeu_pd(mem_addr, element_mask8(e), a); +#endif + } +} + +#endif //PERNIX_AVX512_COMPAT_H diff --git a/src/internal/pernix/x86/avx512vbmi/packing.h b/src/internal/pernix/x86/avx512vbmi/packing.h new file mode 100644 index 0000000..e51c6cc --- /dev/null +++ b/src/internal/pernix/x86/avx512vbmi/packing.h @@ -0,0 +1,331 @@ +#ifndef PERNIX_AVX512VBMI_PACKING_H +#define PERNIX_AVX512VBMI_PACKING_H + +#include +#include + +namespace pernix::internal { + namespace m128 { + /** + * @brief Pack 8 16-bit values for bit widths 9 through 16 using VBMI. + */ + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i &input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = pack_tables_avx512_16; + const __m128i maskv = _mm_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); + const __m128i masked = _mm_and_si128(input, maskv); + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m128i permuted1 = _mm_permutexvar_epi16(tables::get_permute1(), masked); + const __m128i permuted2 = _mm_permutexvar_epi16(tables::get_permute2(), masked); + + const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); + const __m128i shifted2 = _mm_srlv_epi16(permuted2, tables::get_shift2()); + + return _mm_or_si128(shifted1, shifted2); + } else { + const auto [mask1, mask2, mask3] = tables::get_permute_masks(); + + const __m128i permuted1 = _mm_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); + const __m128i permuted2 = _mm_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); + const __m128i permuted3 = _mm_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); + + const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); + const __m128i shifted3 = _mm_srlv_epi16(permuted3, tables::get_shift3()); + + return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); + } + } + } + + /** + * @brief Pack 16 8-bit values for bit widths 1 through 8 using VBMI. + */ + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i &input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + const __m128i maskv = _mm_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); + const __m128i masked = _mm_and_si128(input, maskv); + + if constexpr (BIT_WIDTH == 1) { + return _mm_set1_epi16(static_cast(_mm_cmpgt_epi8_mask(masked, _mm_setzero_si128()))); + } else if constexpr (BIT_WIDTH == 2) { + const __m128i shifted = _mm_srli_epi16(masked, 6); + const __m128i combined = _mm_or_si128(masked, shifted); + + const __m128i shifted2 = _mm_srli_epi32(combined, 12); + const __m128i combined2 = _mm_or_si128(shifted2, combined); + + return _mm_cvtepi32_epi8(combined2); + } else if constexpr (BIT_WIDTH == 3) { + const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); + const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); + + const __m128i pair6 = _mm_or_si128(even, _mm_srli_epi16(odd, 5)); + const __m128i packed12 = _mm_or_si128(pair6, _mm_srli_epi32(pair6, 10)); + + return m128::mm_pack_epi16_avx512vbmi_9to16<12>(_mm_cvtepi32_epi16(packed12)); + } else if constexpr (BIT_WIDTH == 4) { + const __m128i shifted = _mm_srli_epi16(masked, 4); + const __m128i combined = _mm_or_si128(masked, shifted); + + return _mm_cvtepi16_epi8(combined); + } else { + const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); + const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); + + const __m128i shifted = _mm_or_si128(even, _mm_srli_epi16(odd, 8 - BIT_WIDTH)); + return mm_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); + } + } + } + + /** + * @brief Pack 4 32-bit values for bit widths 17 through 24 using VBMI. + */ + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i &input) { + using tables = pack_tables_avx512_24; + + const __m128i maskv = _mm_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); + const __m128i masked = _mm_and_si128(input, maskv); + + const __m128 permuted1 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute1()); + const __m128 permuted2 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute2()); + const __m128 permuted3 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute3()); + + const __m128i shifted1 = _mm_sllv_epi32(_mm_castps_si128(permuted1), tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi32(_mm_castps_si128(permuted2), tables::get_shift2()); + const __m128i shifted3 = _mm_srlv_epi32(_mm_castps_si128(permuted3), tables::get_shift3()); + + return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); + } + } // namespace m128 + + namespace m256 { + /** + * @brief Pack 16 16-bit values for bit widths 9 through 16 using VBMI. + */ + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i &input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = pack_tables_avx512_16; + const __m256i maskv = _mm256_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); + const __m256i masked = _mm256_and_si256(input, maskv); + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m256i permuted1 = _mm256_permutexvar_epi16(tables::get_permute1(), masked); + const __m256i permuted2 = _mm256_permutexvar_epi16(tables::get_permute2(), masked); + + const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); + const __m256i shifted2 = _mm256_srlv_epi16(permuted2, tables::get_shift2()); + + return _mm256_or_si256(shifted1, shifted2); + } else { + const auto [mask1, mask2, mask3] = tables::get_permute_masks(); + + const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); + const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); + const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); + + const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); + const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); + const __m256i shifted3 = _mm256_srlv_epi16(permuted3, tables::get_shift3()); + + return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); + } + } + } + + /** + * @brief Pack 32 8-bit values for bit widths 1 through 8 using VBMI. + */ + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i &input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + const __m256i maskv = _mm256_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); + const __m256i masked = _mm256_and_si256(input, maskv); + + if constexpr (BIT_WIDTH == 1) { + return _mm256_set1_epi32( + static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); + } else if constexpr (BIT_WIDTH == 2) { + const __m256i shifted = _mm256_srli_epi16(masked, 6); + const __m256i combined = _mm256_or_si256(masked, shifted); + + const __m256i shifted2 = _mm256_srli_epi32(combined, 12); + const __m256i combined2 = _mm256_or_si256(shifted2, combined); + + return _mm256_castsi128_si256(_mm256_cvtepi32_epi8(combined2)); + } else if constexpr (BIT_WIDTH == 3) { + const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); + const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); + + const __m256i pair6 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 5)); + const __m256i packed12 = _mm256_or_si256(pair6, _mm256_srli_epi32(pair6, 10)); + + return m256::mm256_pack_epi16_avx512vbmi_9to16<12>( + _mm256_castsi128_si256(_mm256_cvtepi32_epi16(packed12))); + } else if constexpr (BIT_WIDTH == 4) { + const __m256i shifted = _mm256_srli_epi16(masked, 4); + const __m256i combined = _mm256_or_si256(masked, shifted); + + return _mm256_castsi128_si256(_mm256_cvtepi16_epi8(combined)); + } else { + const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); + const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); + + const __m256i shifted = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); + return mm256_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); + } + } + } + + /** + * @brief Pack 8 32-bit values for bit widths 17 through 24 using VBMI. + */ + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i &input) { + using tables = pack_tables_avx512_24; + + const __m256i maskv = _mm256_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); + const __m256i masked = _mm256_and_si256(input, maskv); + + const __m256i permuted1 = _mm256_permutexvar_epi32(tables::get_permute1(), masked); + const __m256i permuted2 = _mm256_permutexvar_epi32(tables::get_permute2(), masked); + const __m256i permuted3 = _mm256_permutexvar_epi32(tables::get_permute3(), masked); + + const __m256i shifted1 = _mm256_sllv_epi32(permuted1, tables::get_shift1()); + const __m256i shifted2 = _mm256_sllv_epi32(permuted2, tables::get_shift2()); + const __m256i shifted3 = _mm256_srlv_epi32(permuted3, tables::get_shift3()); + + return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); + } + } // namespace m256 + + namespace m512 { + /** + * @brief Pack 32 16-bit values for bit widths 9 through 16 using VBMI. + */ + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i &input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = pack_tables_avx512_16; + const __m512i maskv = _mm512_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); + const __m512i masked = _mm512_and_si512(input, maskv); + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m512i permuted1 = _mm512_permutexvar_epi16(tables::get_permute1(), masked); + const __m512i permuted2 = _mm512_permutexvar_epi16(tables::get_permute2(), masked); + + const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); + const __m512i shifted2 = _mm512_srlv_epi16(permuted2, tables::get_shift2()); + + return _mm512_or_si512(shifted1, shifted2); + } else { + const auto [mask1, mask2, mask3] = tables::get_permute_masks(); + + const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); + const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); + const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); + + const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); + const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); + const __m512i shifted3 = _mm512_srlv_epi16(permuted3, tables::get_shift3()); + + return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); + } + } + } + + /** + * @brief Pack 64 8-bit values for bit widths 1 through 8 using VBMI. + */ + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i &input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + const __m512i maskv = _mm512_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); + const __m512i masked = _mm512_and_si512(input, maskv); + + if constexpr (BIT_WIDTH == 1) { + return _mm512_set1_epi64( + static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); + } else if constexpr (BIT_WIDTH == 2) { + const __m512i shifted = _mm512_srli_epi16(masked, 6); + const __m512i combined = _mm512_or_si512(masked, shifted); + + const __m512i shifted2 = _mm512_srli_epi32(combined, 12); + const __m512i combined2 = _mm512_or_si512(shifted2, combined); + + return _mm512_castsi128_si512(_mm512_cvtepi32_epi8(combined2)); + } else if constexpr (BIT_WIDTH == 3) { + const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); + const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); + + const __m512i pair6 = _mm512_or_si512(even, _mm512_srli_epi16(odd, 5)); + const __m512i packed12 = _mm512_or_si512(pair6, _mm512_srli_epi32(pair6, 10)); + + return _mm512_castsi256_si512( + m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm512_cvtepi32_epi16(packed12))); + } else if constexpr (BIT_WIDTH == 4) { + const __m512i shifted = _mm512_srli_epi16(masked, 4); + const __m512i combined = _mm512_or_si512(masked, shifted); + + return _mm512_castsi256_si512(_mm512_cvtepi16_epi8(combined)); + } else { + const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); + const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); + + const __m512i shifted = _mm512_or_si512(even, _mm512_srli_epi16(odd, 8 - BIT_WIDTH)); + return mm512_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); + } + } + } + + /** + * @brief Pack 16 32-bit values for bit widths 17 through 24 using VBMI. + */ + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i &input) { + using tables = pack_tables_avx512_24; + + const __m512i maskv = _mm512_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); + const __m512i masked = _mm512_and_si512(input, maskv); + + const __m512i permuted1 = _mm512_permutexvar_epi32(tables::get_permute1(), masked); + const __m512i permuted2 = _mm512_permutexvar_epi32(tables::get_permute2(), masked); + const __m512i permuted3 = _mm512_permutexvar_epi32(tables::get_permute3(), masked); + + const __m512i shifted1 = _mm512_sllv_epi32(permuted1, tables::get_shift1()); + const __m512i shifted2 = _mm512_sllv_epi32(permuted2, tables::get_shift2()); + const __m512i shifted3 = _mm512_srlv_epi32(permuted3, tables::get_shift3()); + + return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); + } + } // namespace m512 +} // namespace pernix::internal + +#endif // PERNIX_AVX512VBMI_PACKING_H diff --git a/include/pernix/x86/avx512vbmi/tables.h b/src/internal/pernix/x86/avx512vbmi/tables.h similarity index 51% rename from include/pernix/x86/avx512vbmi/tables.h rename to src/internal/pernix/x86/avx512vbmi/tables.h index 4d66727..ab3c665 100644 --- a/include/pernix/x86/avx512vbmi/tables.h +++ b/src/internal/pernix/x86/avx512vbmi/tables.h @@ -9,22 +9,22 @@ #include namespace pernix::internal { -template -static __always_inline Vec load_table(const std::array& table) { - static_assert(sizeof(table) >= sizeof(Vec), "table is smaller than requested SIMD vector"); - if constexpr (std::is_same_v) { - return _mm512_load_si512(static_cast(table.data())); - } else if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(table.data())); - } else { - return _mm_load_si128(reinterpret_cast(table.data())); + template + static __always_inline Vec load_table(const std::array &table) { + static_assert(sizeof(table) >= sizeof(Vec), "table is smaller than requested SIMD vector"); + if constexpr (std::is_same_v) { + return _mm512_load_si512(static_cast(table.data())); + } else if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(table.data())); + } else { + return _mm_load_si128(reinterpret_cast(table.data())); + } } -} -template - requires(N >= 9 && N <= 15) -struct pack_tables_avx512_16 { - alignas(64) inline static constexpr std::array permute1 = [] { + template + requires(N >= 9 && N <= 15) + struct pack_tables_avx512_16 { + alignas(64) inline static constexpr std::array permute1 = [] { // clang-format off if constexpr (N == 9) { return std::array{ @@ -112,10 +112,10 @@ struct pack_tables_avx512_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute2 = [] { + alignas(64) inline static constexpr std::array permute2 = [] { // clang-format off if constexpr (N == 9) { return std::array{ @@ -202,10 +202,10 @@ struct pack_tables_avx512_16 { 29, 30, -1, -1 }; } - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute3 = [] { + alignas(64) inline static constexpr std::array permute3 = [] { // clang-format off if constexpr (N == 9) { return std::array{ @@ -257,10 +257,10 @@ struct pack_tables_avx512_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift1 = [] { + alignas(64) inline static constexpr std::array shift1 = [] { // clang-format off if constexpr (N == 9) { return std::array{ @@ -348,10 +348,10 @@ struct pack_tables_avx512_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift2 = [] { + alignas(64) inline static constexpr std::array shift2 = [] { // clang-format off if constexpr (N == 9) { return std::array{ @@ -439,10 +439,10 @@ struct pack_tables_avx512_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift3 = [] { + alignas(64) inline static constexpr std::array shift3 = [] { // clang-format off if constexpr (N == 9) { return std::array{ @@ -494,10 +494,10 @@ struct pack_tables_avx512_16 { }; } return std::array{}; - // clang-format on - }(); + // clang-format on + }(); - inline static constexpr std::tuple<__mmask32, __mmask32, __mmask32> get_permute_masks() { + inline static constexpr std::tuple<__mmask32, __mmask32, __mmask32> get_permute_masks() { // clang-format off if constexpr (N == 9) { return { @@ -525,296 +525,305 @@ struct pack_tables_avx512_16 { }; } return {0, 0, 0}; - // clang-format on - } - - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } - static __always_inline Vec get_permute3() { return load_table(permute3); } - - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } - static __always_inline Vec get_shift3() { return load_table(shift3); } -}; - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct pack_tables_avx512_24 { -private: - struct word_plan { - int32_t left_index1 = -1; - int32_t left_index2 = -1; - int32_t right_index = -1; - uint32_t left_shift1 = 32; - uint32_t left_shift2 = 32; - uint32_t right_shift = 32; - }; + // clang-format on + } - static constexpr word_plan create_plan(const uint32_t idx) { - word_plan plan{}; + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } + static __always_inline Vec get_permute3() { return load_table(permute3); } - const uint32_t word_start = idx * 32u; - const uint32_t word_end = word_start + 32u; + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } + static __always_inline Vec get_shift3() { return load_table(shift3); } + }; - uint32_t left_slot = 0; - for (uint32_t input_lane = 0; input_lane < 16; ++input_lane) { - const uint32_t input_start = input_lane * BIT_WIDTH; - const uint32_t input_end = input_start + BIT_WIDTH; + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && + (std::is_same_v || std::is_same_v || std::is_same_v)) + struct pack_tables_avx512_24 { + private: + struct word_plan { + int32_t left_index1 = -1; + int32_t left_index2 = -1; + int32_t right_index = -1; + uint32_t left_shift1 = 32; + uint32_t left_shift2 = 32; + uint32_t right_shift = 32; + }; + + static constexpr word_plan create_plan(const uint32_t idx) { + word_plan plan{}; + + const uint32_t word_start = idx * 32u; + const uint32_t word_end = word_start + 32u; + + uint32_t left_slot = 0; + for (uint32_t input_lane = 0; input_lane < 16; ++input_lane) { + const uint32_t input_start = input_lane * BIT_WIDTH; + const uint32_t input_end = input_start + BIT_WIDTH; + + const uint32_t overlap_start = std::max(word_start, input_start); + const uint32_t overlap_end = std::min(word_end, input_end); + if (overlap_start >= overlap_end) { + continue; + } - const uint32_t overlap_start = std::max(word_start, input_start); - const uint32_t overlap_end = std::min(word_end, input_end); - if (overlap_start >= overlap_end) { - continue; + const auto output_bit = static_cast(overlap_start - word_start); + const auto input_bit = static_cast(overlap_start - input_start); + const int32_t delta = output_bit - input_bit; + + if (delta >= 0) { + if (left_slot == 0) { + plan.left_index1 = static_cast(input_lane); + plan.left_shift1 = static_cast(delta); + ++left_slot; + } else { + plan.left_index2 = static_cast(input_lane); + plan.left_shift2 = static_cast(delta); + } + } else { + plan.right_index = static_cast(input_lane); + plan.right_shift = static_cast(-delta); + } } - const auto output_bit = static_cast(overlap_start - word_start); - const auto input_bit = static_cast(overlap_start - input_start); - const int32_t delta = output_bit - input_bit; + return plan; + } - if (delta >= 0) { - if (left_slot == 0) { - plan.left_index1 = static_cast(input_lane); - plan.left_shift1 = static_cast(delta); - ++left_slot; - } else { - plan.left_index2 = static_cast(input_lane); - plan.left_shift2 = static_cast(delta); - } - } else { - plan.right_index = static_cast(input_lane); - plan.right_shift = static_cast(-delta); + static constexpr std::array word_plans = [] { + std::array plans{}; + for (uint32_t i = 0; i < 16; ++i) { + plans[i] = create_plan(i); + } + return plans; + }(); + + template + static __always_inline constexpr std::array make_table(Getter getter) { + std::array values{}; + for (uint32_t i = 0; i < 16; ++i) { + values[i] = getter(word_plans[i]); } + return values; } - return plan; - } + alignas(64) static constexpr auto permute1 = make_table([](const word_plan &p) { + return p.left_index1; + }); - static constexpr std::array word_plans = [] { - std::array plans{}; - for (uint32_t i = 0; i < 16; ++i) { - plans[i] = create_plan(i); - } - return plans; - }(); - - template - static __always_inline constexpr std::array make_table(Getter getter) { - std::array values{}; - for (uint32_t i = 0; i < 16; ++i) { - values[i] = getter(word_plans[i]); - } - return values; - } + alignas(64) static constexpr auto permute2 = make_table([](const word_plan &p) { + return p.left_index2; + }); - alignas(64) static constexpr auto permute1 = make_table([](const word_plan& p) { return p.left_index1; }); + alignas(64) static constexpr auto permute3 = make_table([](const word_plan &p) { + return p.right_index; + }); - alignas(64) static constexpr auto permute2 = make_table([](const word_plan& p) { return p.left_index2; }); + alignas(64) static constexpr auto shift1 = make_table( + [](const word_plan &p) { return p.left_shift1; }); - alignas(64) static constexpr auto permute3 = make_table([](const word_plan& p) { return p.right_index; }); + alignas(64) static constexpr auto shift2 = make_table( + [](const word_plan &p) { return p.left_shift2; }); - alignas(64) static constexpr auto shift1 = make_table([](const word_plan& p) { return p.left_shift1; }); + alignas(64) static constexpr auto shift3 = make_table( + [](const word_plan &p) { return p.right_shift; }); - alignas(64) static constexpr auto shift2 = make_table([](const word_plan& p) { return p.left_shift2; }); + public: + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } + static __always_inline Vec get_permute3() { return load_table(permute3); } - alignas(64) static constexpr auto shift3 = make_table([](const word_plan& p) { return p.right_shift; }); + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } + static __always_inline Vec get_shift3() { return load_table(shift3); } + }; -public: - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } - static __always_inline Vec get_permute3() { return load_table(permute3); } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && + (std::is_same_v || std::is_same_v || std::is_same_v)) + struct unpack_tables_avx512_8 { + private: + alignas(64) inline static constexpr std::array permute1 = [] { + std::array table{}; + std::ranges::fill(table, -1); + for (size_t entry = 0; entry < 64; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t first_byte = bit_start / 8; + + table[entry] = static_cast(first_byte); + } - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } - static __always_inline Vec get_shift3() { return load_table(shift3); } -}; + return table; + }(); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct unpack_tables_avx512_8 { -private: - alignas(64) inline static constexpr std::array permute1 = [] { - std::array table{}; - std::ranges::fill(table, -1); - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; + alignas(64) inline static constexpr std::array permute2 = [] { + std::array table{}; + std::ranges::fill(table, -1); - table[entry] = static_cast(first_byte); - } + for (size_t entry = 0; entry < 64; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t first_byte = bit_start / 8; + const size_t bit_offset = bit_start % 8; - return table; - }(); + if (bit_offset + BIT_WIDTH > 8) { + table[entry] = static_cast(first_byte + 1); + } + } - alignas(64) inline static constexpr std::array permute2 = [] { - std::array table{}; - std::ranges::fill(table, -1); + return table; + }(); - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t bit_offset = bit_start % 8; + alignas(64) inline static constexpr std::array shift1 = [] { + std::array table{}; - if (bit_offset + BIT_WIDTH > 8) { - table[entry] = static_cast(first_byte + 1); - } - } + for (size_t entry = 0; entry < 64; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_offset = bit_start % 8; - return table; - }(); + table[entry] = static_cast(bit_offset); + } - alignas(64) inline static constexpr std::array shift1 = [] { - std::array table{}; + return table; + }(); - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8; + alignas(64) inline static constexpr std::array shift2 = [] { + std::array table{}; - table[entry] = static_cast(bit_offset); - } + for (size_t entry = 0; entry < 64; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_offset = bit_start % 8u; + const size_t spill_bits = (bit_offset + BIT_WIDTH > 8u) ? (bit_offset + BIT_WIDTH - 8u) : 0u; - return table; - }(); + table[entry] = spill_bits ? static_cast(8 - bit_offset) : 0; + } - alignas(64) inline static constexpr std::array shift2 = [] { - std::array table{}; + return table; + }(); - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - const size_t spill_bits = (bit_offset + BIT_WIDTH > 8u) ? (bit_offset + BIT_WIDTH - 8u) : 0u; + public: + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } - table[entry] = spill_bits ? static_cast(8 - bit_offset) : 0; - } + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } + }; - return table; - }(); - -public: - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } - - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } -}; - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct unpack_tables_avx512_16 { -private: - alignas(64) inline static constexpr std::array permute1 = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t base = entry * 2; - - table[base] = static_cast(first_byte); - table[base + 1] = static_cast(first_byte + 1); - } + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && + (std::is_same_v || std::is_same_v || std::is_same_v)) + struct unpack_tables_avx512_16 { + private: + alignas(64) inline static constexpr std::array permute1 = [] { + std::array table{}; + std::ranges::fill(table, -1); + + for (size_t entry = 0; entry < 32; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t first_byte = bit_start / 8; + const size_t base = entry * 2; + + table[base] = static_cast(first_byte); + table[base + 1] = static_cast(first_byte + 1); + } - return table; - }(); + return table; + }(); - alignas(64) inline static constexpr std::array permute2 = [] { - std::array table{}; - std::ranges::fill(table, -1); + alignas(64) inline static constexpr std::array permute2 = [] { + std::array table{}; + std::ranges::fill(table, -1); - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t bit_offset = bit_start % 8; - const size_t base = entry * 2; + for (size_t entry = 0; entry < 32; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t first_byte = bit_start / 8; + const size_t bit_offset = bit_start % 8; + const size_t base = entry * 2; - if (bit_offset + BIT_WIDTH > 16) { - table[base] = static_cast(first_byte + 2); + if (bit_offset + BIT_WIDTH > 16) { + table[base] = static_cast(first_byte + 2); + } } - } - return table; - }(); + return table; + }(); - alignas(64) inline static constexpr std::array shift1 = [] { - std::array table{}; + alignas(64) inline static constexpr std::array shift1 = [] { + std::array table{}; - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; + for (size_t entry = 0; entry < 32; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_offset = bit_start % 8u; - // Right-shift the 16-bit chunk so the value starts at bit 0. - table[entry] = static_cast(bit_offset); - } + // Right-shift the 16-bit chunk so the value starts at bit 0. + table[entry] = static_cast(bit_offset); + } - return table; - }(); + return table; + }(); - alignas(64) inline static constexpr std::array shift2 = [] { - std::array table{}; - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - const size_t spill_bits = (bit_offset + BIT_WIDTH > 16u) ? (bit_offset + BIT_WIDTH - 16u) : 0u; + alignas(64) inline static constexpr std::array shift2 = [] { + std::array table{}; + for (size_t entry = 0; entry < 32; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_offset = bit_start % 8u; + const size_t spill_bits = (bit_offset + BIT_WIDTH > 16u) ? (bit_offset + BIT_WIDTH - 16u) : 0u; - // Move spill bits from byte3 to their final bit positions before merge. - table[entry] = spill_bits ? static_cast(16u - bit_offset) : 0; - } + // Move spill bits from byte3 to their final bit positions before merge. + table[entry] = spill_bits ? static_cast(16u - bit_offset) : 0; + } - return table; - }(); + return table; + }(); -public: - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } + public: + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } -}; + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } + }; -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct unpack_tables_avx512_24 { -private: - alignas(64) inline static constexpr std::array permute = [] { - std::array table{}; - std::ranges::fill(table, -1); + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && + (std::is_same_v || std::is_same_v || std::is_same_v)) + struct unpack_tables_avx512_24 { + private: + alignas(64) inline static constexpr std::array permute = [] { + std::array table{}; + std::ranges::fill(table, -1); - for (size_t entry = 0; entry < 16; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_end = bit_start + BIT_WIDTH - 1; + for (size_t entry = 0; entry < 16; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_end = bit_start + BIT_WIDTH - 1; - const size_t first_byte = bit_start / 8; - const size_t last_byte = bit_end / 8; + const size_t first_byte = bit_start / 8; + const size_t last_byte = bit_end / 8; - const size_t base = entry * 4; + const size_t base = entry * 4; - for (size_t byte = first_byte; byte <= last_byte; ++byte) { - table[base + (byte - first_byte)] = static_cast(byte); + for (size_t byte = first_byte; byte <= last_byte; ++byte) { + table[base + (byte - first_byte)] = static_cast(byte); + } } - } - return table; - }(); + return table; + }(); - alignas(64) inline static constexpr std::array shift = [] { - std::array table{}; + alignas(64) inline static constexpr std::array shift = [] { + std::array table{}; - for (size_t entry = 0; entry < 16; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - table[entry] = static_cast(32u - BIT_WIDTH - (bit_start % 8u)); - } + for (size_t entry = 0; entry < 16; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + table[entry] = static_cast(32u - BIT_WIDTH - (bit_start % 8u)); + } - return table; - }(); + return table; + }(); -public: - static __always_inline Vec get_permute() { return load_table(permute); } - static __always_inline Vec get_shift() { return load_table(shift); } -}; + public: + static __always_inline Vec get_permute() { return load_table(permute); } + static __always_inline Vec get_shift() { return load_table(shift); } + }; } // namespace pernix::internal #endif // PERNIX_AVX512VBMI_TABLES_H diff --git a/src/internal/pernix/x86/avx512vbmi/unpacking.h b/src/internal/pernix/x86/avx512vbmi/unpacking.h new file mode 100644 index 0000000..6799cd2 --- /dev/null +++ b/src/internal/pernix/x86/avx512vbmi/unpacking.h @@ -0,0 +1,500 @@ +#ifndef PERNIX_AVX512VBMI_UNPACKING_H +#define PERNIX_AVX512VBMI_UNPACKING_H + +#include +#include + +namespace pernix::internal { + namespace m128 { + constexpr __mmask16 kAlternateByteMask16 = 0xAAAAULL; + +__always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i count) { + const __m128i mask = _mm_set1_epi16(0x00ff); + const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); + const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); + return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); + } + +__always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { + const __m128i mask = _mm_set1_epi16(0xff00); + const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); + const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); + return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); + } + +__always_inline static __m128i _mm_slli_epi8(const __m128i a, const int8_t imm8) { + return _mm_sllv_epi8(a, _mm_set1_epi8(imm8)); + } + +__always_inline static __m128i _mm_srli_epi8(const __m128i a, const int imm8) { + const __m128i lo_mask = _mm_set1_epi16(0x00ff); + const __m128i hi_mask = _mm_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); + + const __m128i lo = _mm_srl_epi16(_mm_and_si128(a, lo_mask), shift); + const __m128i hi = _mm_and_si128(_mm_srl_epi16(a, shift), hi_mask); + + return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); + } + +__always_inline static __m128i _mm_srai_epi8(const __m128i a, const int8_t imm8) { + const __m128i lo_mask = _mm_set1_epi16(0x00ff); + const __m128i hi_mask = _mm_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); + + const __m128i hi = _mm_and_si128(_mm_sra_epi16(a, shift), hi_mask); + + const __m128i lo_as_hi = _mm_slli_epi16(_mm_and_si128(a, lo_mask), 8); + const __m128i lo = _mm_and_si128(_mm_srli_epi16(_mm_sra_epi16(lo_as_hi, shift), 8), lo_mask); + + return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); + } + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i &input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + if constexpr (BIT_WIDTH == 1) { + const auto value = static_cast<__mmask16>(_mm_cvtsi128_si64(input)); + const __m128i source = _mm_movm_epi8(value); + const __m128i unpacked = _mm_abs_epi8(source); + return unpacked; + } else if constexpr (BIT_WIDTH == 2) { + __m128i values_shift0 = input; + __m128i values_shift2 = _mm_srli_epi16(values_shift0, 2); + const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); + const __m128i values_shift6 = _mm_srli_epi16(values_shift0, 6); + + __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift2); + values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift2); + values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); + + interleave_tmp = _mm_unpacklo_epi8(values_shift4, values_shift6); + values_shift2 = _mm_unpackhi_epi8(values_shift4, values_shift6); + values_shift2 = _mm_unpacklo_epi64(interleave_tmp, values_shift2); + + interleave_tmp = _mm_unpacklo_epi16(values_shift0, values_shift2); + values_shift0 = _mm_unpackhi_epi16(values_shift0, values_shift2); + values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); + values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); + + values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0303)); + + return values_shift0; + } else if constexpr (BIT_WIDTH == 4) { + __m128i values_shift0 = input; + const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); + + const __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift4); + values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift4); + values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); + values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); + + values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0F0F)); + + return values_shift0; + } else { + using tables = unpack_tables_avx512_8; + + const __m128i permuted1 = _mm_permutexvar_epi8(tables::get_permute1(), input); + const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); + + const __m128i shifted1 = _mm_srlv_epi8(permuted1, tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi8(permuted2, tables::get_shift2()); + + const __mmask16 spill_mask = _mm_cmpneq_epi8_mask(tables::get_shift2(), _mm_setzero_si128()); + __m128i combined = _mm_or_si128(shifted1, _mm_maskz_mov_epi8(spill_mask, shifted2)); + + constexpr uint32_t shift = 8 - BIT_WIDTH; + combined = _mm_slli_epi8(combined, shift); + if (SIGN_VALUES) { + combined = _mm_srai_epi8(combined, shift); + } else { + combined = _mm_srli_epi8(combined, shift); + } + + return combined; + } + } + } + + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i &input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = unpack_tables_avx512_16; + + const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute1(), input); + + __m128i shifted = _mm_srlv_epi16(permuted, tables::get_shift1()); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); + const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); + shifted = _mm_or_si128(shifted, shifted2); + } + + constexpr uint32_t shift = 16 - BIT_WIDTH; + shifted = _mm_slli_epi16(shifted, shift); + if (SIGN_VALUES) { + shifted = _mm_srai_epi16(shifted, shift); + } else { + shifted = _mm_srli_epi16(shifted, shift); + } + + return shifted; + } + } + + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i &input) { + using tables = unpack_tables_avx512_24; + + const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute(), input); + + constexpr uint32_t shift = 32 - BIT_WIDTH; + __m128i shifted = _mm_sllv_epi32(permuted, tables::get_shift()); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + shifted = _mm_srai_epi32(shifted, shift); + } else { + shifted = _mm_srli_epi32(shifted, shift); + } + + return shifted; + } + } // namespace m128 + + namespace m256 { + constexpr __mmask32 kAlternateByteMask32 = 0xAAAAAAAAULL; + +__always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i count) { + const __m256i mask = _mm256_set1_epi16(0x00ff); + const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); + const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); + return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); + } + +__always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { + const __m256i mask = _mm256_set1_epi16(0xff00); + const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); + const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); + return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); + } + +__always_inline static __m256i _mm256_slli_epi8(const __m256i a, const int8_t imm8) { + return _mm256_sllv_epi8(a, _mm256_set1_epi8(imm8)); + } + +__always_inline static __m256i _mm256_srli_epi8(const __m256i a, const int8_t imm8) { + const __m256i lo_mask = _mm256_set1_epi16(0x00ff); + const __m256i hi_mask = _mm256_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); + + const __m256i lo = _mm256_srl_epi16(_mm256_and_si256(a, lo_mask), shift); + const __m256i hi = _mm256_and_si256(_mm256_srl_epi16(a, shift), hi_mask); + + return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); + } + +__always_inline static __m256i _mm256_srai_epi8(const __m256i a, const int8_t imm8) { + const __m256i lo_mask = _mm256_set1_epi16(0x00ff); + const __m256i hi_mask = _mm256_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); + + const __m256i hi = _mm256_and_si256(_mm256_sra_epi16(a, shift), hi_mask); + + const __m256i lo_as_hi = _mm256_slli_epi16(_mm256_and_si256(a, lo_mask), 8); + const __m256i lo = _mm256_and_si256(_mm256_srli_epi16(_mm256_sra_epi16(lo_as_hi, shift), 8), lo_mask); + + return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); + } + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i &input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + if constexpr (BIT_WIDTH == 1) { + const auto value = static_cast<__mmask32>(_mm_cvtsi128_si64(_mm256_castsi256_si128(input))); + const __m256i source = _mm256_movm_epi8(value); + const __m256i unpacked = _mm256_abs_epi8(source); + return unpacked; + } else if constexpr (BIT_WIDTH == 2) { + __m256i values_shift0 = input; + __m256i values_shift2 = _mm256_srli_epi16(values_shift0, 2); + const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); + const __m256i values_shift6 = _mm256_srli_epi16(values_shift0, 6); + + __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift2); + values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift2); + values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); + + interleave_tmp = _mm256_unpacklo_epi8(values_shift4, values_shift6); + values_shift2 = _mm256_unpackhi_epi8(values_shift4, values_shift6); + values_shift2 = _mm256_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); + + interleave_tmp = _mm256_unpacklo_epi16(values_shift0, values_shift2); + values_shift0 = _mm256_unpackhi_epi16(values_shift0, values_shift2); + values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); + values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); + + values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0303)); + + return values_shift0; + } else if constexpr (BIT_WIDTH == 4) { + __m256i values_shift0 = input; + const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); + + __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift4); + values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift4); + values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); + values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); + + values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0F0F)); + + return values_shift0; + } else { + using tables = unpack_tables_avx512_8; + + const __m256i permuted1 = _mm256_permutexvar_epi8(tables::get_permute1(), input); + const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); + + const __m256i shifted1 = _mm256_srlv_epi8(permuted1, tables::get_shift1()); + const __m256i shifted2 = _mm256_sllv_epi8(permuted2, tables::get_shift2()); + + const __mmask32 spill_mask = _mm256_cmpneq_epi8_mask(tables::get_shift2(), _mm256_setzero_si256()); + __m256i combined = _mm256_or_si256(shifted1, _mm256_maskz_mov_epi8(spill_mask, shifted2)); + + constexpr uint32_t shift = 8 - BIT_WIDTH; + combined = _mm256_slli_epi8(combined, shift); + if (SIGN_VALUES) { + combined = _mm256_srai_epi8(combined, shift); + } else { + combined = _mm256_srli_epi8(combined, shift); + } + + return combined; + } + } + } + + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i &input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = unpack_tables_avx512_16; + + const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute1(), input); + + __m256i shifted = _mm256_srlv_epi16(permuted, tables::get_shift1()); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); + const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); + shifted = _mm256_or_si256(shifted, shifted2); + } + + constexpr uint32_t shift = 16 - BIT_WIDTH; + shifted = _mm256_slli_epi16(shifted, shift); + if (SIGN_VALUES) { + shifted = _mm256_srai_epi16(shifted, shift); + } else { + shifted = _mm256_srli_epi16(shifted, shift); + } + + return shifted; + } + } + + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i &input) { + using tables = unpack_tables_avx512_24; + + const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute(), input); + + constexpr uint32_t shift = 32 - BIT_WIDTH; + __m256i shifted = _mm256_sllv_epi32(permuted, tables::get_shift()); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + shifted = _mm256_srai_epi32(shifted, shift); + } else { + shifted = _mm256_srli_epi32(shifted, shift); + } + + return shifted; + } + } // namespace m256 + + namespace m512 { + constexpr __mmask64 kAlternateByteMask64 = 0xAAAAAAAAAAAAAAAAULL; + +__always_inline static __m512i _mm512_srlv_epi8(const __m512i a, const __m512i count) { + const __m512i mask = _mm512_set1_epi16(0x00ff); + const __m512i low_half = _mm512_srlv_epi16(_mm512_and_si512(mask, a), _mm512_and_si512(mask, count)); + const __m512i high_half = _mm512_srlv_epi16(a, _mm512_srli_epi16(count, 8)); + return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); + } + +__always_inline static __m512i _mm512_sllv_epi8(const __m512i a, const __m512i count) { + const __m512i mask = _mm512_set1_epi16(0xff00); + const __m512i low_half = _mm512_sllv_epi16(a, _mm512_andnot_si512(mask, count)); + const __m512i high_half = _mm512_sllv_epi16(_mm512_and_si512(mask, a), _mm512_srli_epi16(count, 8)); + return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); + } + +__always_inline static __m512i _mm512_slli_epi8(const __m512i a, const int8_t imm8) { + return _mm512_sllv_epi8(a, _mm512_set1_epi8(imm8)); + } + +__always_inline static __m512i _mm512_srli_epi8(const __m512i a, const int8_t imm8) { + const __m512i lo_mask = _mm512_set1_epi16(0x00ff); + const __m512i hi_mask = _mm512_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); + + const __m512i lo = _mm512_srl_epi16(_mm512_and_si512(a, lo_mask), shift); + const __m512i hi = _mm512_and_si512(_mm512_srl_epi16(a, shift), hi_mask); + + return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); + } + +__always_inline static __m512i _mm512_srai_epi8(const __m512i a, const int8_t imm8) { + const __m512i lo_mask = _mm512_set1_epi16(0x00ff); + const __m512i hi_mask = _mm512_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); + + const __m512i hi = _mm512_and_si512(_mm512_sra_epi16(a, shift), hi_mask); + + const __m512i lo_as_hi = _mm512_slli_epi16(_mm512_and_si512(a, lo_mask), 8); + const __m512i lo = _mm512_and_si512(_mm512_srli_epi16(_mm512_sra_epi16(lo_as_hi, shift), 8), lo_mask); + + return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); + } + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i &input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + if constexpr (BIT_WIDTH == 1) { + const auto value = static_cast<__mmask64>(_mm_cvtsi128_si64(_mm512_castsi512_si128(input))); + const __m512i source = _mm512_movm_epi8(value); + const __m512i unpacked = _mm512_abs_epi8(source); + return unpacked; + } else if constexpr (BIT_WIDTH == 2) { + __m512i values_shift0 = input; + __m512i values_shift2 = _mm512_srli_epi16(values_shift0, 2); + const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); + const __m512i values_shift6 = _mm512_srli_epi16(values_shift0, 6); + + __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift2); + values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift2); + values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); + + interleave_tmp = _mm512_unpacklo_epi8(values_shift4, values_shift6); + values_shift2 = _mm512_unpackhi_epi8(values_shift4, values_shift6); + values_shift2 = _mm512_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); + + interleave_tmp = _mm512_unpacklo_epi16(values_shift0, values_shift2); + values_shift0 = _mm512_unpackhi_epi16(values_shift0, values_shift2); + values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x88); + values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); + + values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0303)); + + return values_shift0; + } else if constexpr (BIT_WIDTH == 4) { + __m512i values_shift0 = input; + const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); + + __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift4); + values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift4); + values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x44); + values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); + + values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0F0F)); + + return values_shift0; + } else { + using tables = unpack_tables_avx512_8; + + const __m512i permuted1 = _mm512_permutexvar_epi8(tables::get_permute1(), input); + const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); + + const __m512i shifted1 = _mm512_srlv_epi8(permuted1, tables::get_shift1()); + const __m512i shifted2 = _mm512_sllv_epi8(permuted2, tables::get_shift2()); + + const __mmask64 spill_mask = _mm512_cmpneq_epi8_mask(tables::get_shift2(), _mm512_setzero_si512()); + __m512i combined = _mm512_or_si512(shifted1, _mm512_maskz_mov_epi8(spill_mask, shifted2)); + + constexpr uint32_t shift = 8 - BIT_WIDTH; + combined = _mm512_slli_epi8(combined, shift); + if (SIGN_VALUES) { + combined = _mm512_srai_epi8(combined, shift); + } else { + combined = _mm512_srli_epi8(combined, shift); + } + + return combined; + } + } + } + + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i &input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = unpack_tables_avx512_16; + + const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute1(), input); + __m512i shifted = _mm512_srlv_epi16(permuted, tables::get_shift1()); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); + const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); + shifted = _mm512_or_si512(shifted, shifted2); + } + + constexpr uint32_t shift = 16 - BIT_WIDTH; + shifted = _mm512_slli_epi16(shifted, shift); + if (SIGN_VALUES) { + shifted = _mm512_srai_epi16(shifted, shift); + } else { + shifted = _mm512_srli_epi16(shifted, shift); + } + + return shifted; + } + } + + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i &input) { + using tables = unpack_tables_avx512_24; + + const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute(), input); + __m512i shifted = _mm512_sllv_epi32(permuted, tables::get_shift()); + + constexpr uint32_t shift = 32 - BIT_WIDTH; + if constexpr (SIGN_VALUES) { + shifted = _mm512_srai_epi32(shifted, shift); + } else { + shifted = _mm512_srli_epi32(shifted, shift); + } + + return shifted; + } + } // namespace m512 +} // namespace pernix::internal + +#endif // PERNIX_AVX512VBMI_UNPACKING_H diff --git a/include/pernix/x86/bmi2/compression.h b/src/internal/pernix/x86/bmi2/bmi2_compression.h similarity index 56% rename from include/pernix/x86/bmi2/compression.h rename to src/internal/pernix/x86/bmi2/bmi2_compression.h index ca4b976..2f2d362 100644 --- a/include/pernix/x86/bmi2/compression.h +++ b/src/internal/pernix/x86/bmi2/bmi2_compression.h @@ -1,8 +1,8 @@ #ifndef PERNIX_BMI2_COMPRESSION_H #define PERNIX_BMI2_COMPRESSION_H -#include -#include +#include +#include #include #include @@ -12,11 +12,11 @@ namespace pernix { namespace internal { /** - * @brief Build the masks and shift constants used by the BMI2 packers. - * - * @tparam BIT_WIDTH bit width per packed value. - * @return std::tuple mask tuple used by the BMI2 helpers. - */ +* @brief Build the masks and shift constants used by the BMI2 packers. +* +* @tparam BIT_WIDTH bit width per packed value. +* @return std::tuple mask tuple used by the BMI2 helpers. +*/ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) static constexpr std::tuple pack_avx2_bmi2_constants() { @@ -42,12 +42,12 @@ static constexpr std::tuple pack_avx2_bm } /** - * @brief Pack four 32-bit values with BMI2 extract instructions. - * - * @tparam BIT_WIDTH bit width per packed value. - * @param input SIMD register containing four quantized values. - * @return __m128i packed bitstream in the low bytes of the result. - */ +* @brief Pack four 32-bit values with BMI2 extract instructions. +* +* @tparam BIT_WIDTH bit width per packed value. +* @param input SIMD register containing four quantized values. +* @return __m128i packed bitstream in the low bytes of the result. +*/ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { @@ -73,12 +73,12 @@ static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { } /** - * @brief Pack eight 32-bit values with BMI2 extract instructions. - * - * @tparam BIT_WIDTH bit width per packed value. - * @param input SIMD register containing eight quantized values. - * @return __m256i packed bitstream in the low bytes of the result. - */ +* @brief Pack eight 32-bit values with BMI2 extract instructions. +* +* @tparam BIT_WIDTH bit width per packed value. +* @param input SIMD register containing eight quantized values. +* @return __m256i packed bitstream in the low bytes of the result. +*/ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { @@ -144,26 +144,31 @@ static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { append_bits(x2, 2 * chunk_bits); append_bits(x3, 3 * chunk_bits); - return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), static_cast(out2), 0); + return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), + static_cast(out2), 0); } } } // namespace internal /** - * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -template +* @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* +* @param input pointer to the start of the input float values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX2 and BMI2 support. +*/ +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_bmi2(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { +int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const float_t scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; @@ -176,7 +181,7 @@ int mm256_compress_block_bmi2(const float_t* __restrict__ input, const float_t s const __m256 source = _mm256_loadu_ps(input); const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); - const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); + const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); std::memcpy(output, &packed, BIT_WIDTH); input += 8; output += BIT_WIDTH; @@ -187,7 +192,8 @@ int mm256_compress_block_bmi2(const float_t* __restrict__ input, const float_t s #pragma GCC unroll 8 for (uint32_t i = 0; i < remaining; i++) { block_values[i] = - static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], scale))); + static_cast(internal::clamp_signed_quantized( + internal::quantize_ps_epi32(input[i], scale))); } internal::pack_epi32_fallback(block_values, output); @@ -197,19 +203,23 @@ int mm256_compress_block_bmi2(const float_t* __restrict__ input, const float_t s } /** - * @brief Compress a single block of double values using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -template +* @brief Compress a single block of double values using AVX2 and BMI2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @param input pointer to the start of the input double values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX2 and BMI2 support. +*/ +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_bmi2(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { +int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const double_t scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; @@ -225,7 +235,8 @@ int mm256_compress_block_bmi2(const double_t* __restrict__ input, const double_t const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); __m256i combined = _mm256_castsi128_si256(quantized1); combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_bmi2(internal::mm256_clamp_signed_epi32(combined)); + const __m256i packed = internal::mm256_pack_epi32_bmi2( + internal::mm256_clamp_signed_epi32(combined)); std::memcpy(output, &packed, BIT_WIDTH); input += 8; output += BIT_WIDTH; @@ -236,7 +247,8 @@ int mm256_compress_block_bmi2(const double_t* __restrict__ input, const double_t #pragma GCC unroll 8 for (uint32_t i = 0; i < remaining; i++) { block_values[i] = - static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], scale))); + static_cast(internal::clamp_signed_quantized( + internal::quantize_pd_epi64(input[i], scale))); } internal::pack_epi32_fallback(block_values, output); @@ -245,22 +257,25 @@ int mm256_compress_block_bmi2(const double_t* __restrict__ input, const double_t } /** - * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -template +* @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* +* @param input pointer to the start of the input float values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @param blocks number of 512-bit blocks to compress. +* @return int status code (0 for success). +* +* @note This function requires AVX2 and BMI2 support. +*/ +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_bmi2(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { +int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const float_t scale, + void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const float_t* block_input = input; uint8_t* block_output = output; @@ -274,21 +289,24 @@ int mm256_compress_blocks_bmi2(const float_t* __restrict__ input, const float_t } /** - * @brief Compress multiple blocks of double values using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -template +* @brief Compress multiple blocks of double values using AVX2 and BMI2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @param input pointer to the start of the input double values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @param blocks number of blocks to compress. +* @return int status code (0 for success). +* +* @note This function requires AVX2 and BMI2 support. +*/ +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_bmi2(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { +int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const double_t scale, + void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const double_t* block_input = input; uint8_t* block_output = output; @@ -302,70 +320,4 @@ int mm256_compress_blocks_bmi2(const double_t* __restrict__ input, const double_ } } // namespace pernix -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_block_bmi2(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_block_f64_bmi2(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_blocks_bmi2(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_blocks_f64_bmi2(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - #endif // PERNIX_BMI2_COMPRESSION_H diff --git a/include/pernix/x86/bmi2/decompression.h b/src/internal/pernix/x86/bmi2/bmi2_decompression.h similarity index 80% rename from include/pernix/x86/bmi2/decompression.h rename to src/internal/pernix/x86/bmi2/bmi2_decompression.h index 443d673..414d6e0 100644 --- a/include/pernix/x86/bmi2/decompression.h +++ b/src/internal/pernix/x86/bmi2/bmi2_decompression.h @@ -1,7 +1,7 @@ #ifndef PERNIX_BMI2_DECOMPRESSION_H #define PERNIX_BMI2_DECOMPRESSION_H -#include +#include #include #include @@ -205,9 +205,12 @@ __m256i mm256_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { * * @note This function requires AVX2 and BMI2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_bmi2(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { +int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; @@ -246,9 +249,12 @@ int mm256_decompress_block_bmi2(const uint8_t* __restrict__ input, const float_t * * @note This function requires AVX2 and BMI2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_bmi2(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { +int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; @@ -294,10 +300,13 @@ int mm256_decompress_block_bmi2(const uint8_t* __restrict__ input, const double_ * * @note This function requires AVX2 and BMI2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_bmi2(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, +int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const uint8_t* block_input = input; float_t* block_output = output; @@ -325,10 +334,13 @@ int mm256_decompress_blocks_bmi2(const uint8_t* __restrict__ input, const float_ * * @note This function requires AVX2 and BMI2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_bmi2(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, +int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, const uint32_t blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + const uint8_t* block_input = input; double_t* block_output = output; @@ -342,70 +354,4 @@ int mm256_decompress_blocks_bmi2(const uint8_t* __restrict__ input, const double } } // namespace pernix -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Decompress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_decompress_block_bmi2(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_decompress_block_f64_bmi2(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -/** - * @brief Decompress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_decompress_blocks_bmi2(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Decompress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_decompress_blocks_f64_bmi2(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - #endif // PERNIX_BMI2_DECOMPRESSION_H diff --git a/include/pernix/x86/utils.h b/src/internal/pernix/x86/utils.h similarity index 100% rename from include/pernix/x86/utils.h rename to src/internal/pernix/x86/utils.h diff --git a/src/pernix.cpp b/src/pernix.cpp index 94d9d14..8b806a6 100644 --- a/src/pernix.cpp +++ b/src/pernix.cpp @@ -1,232 +1,165 @@ #include +#include -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -// Use the best available implementation based on detected CPU features at compile time -#if defined(PERNIX_BACKEND_X86) && defined(PERNIX_AVX2_ENABLED) -#ifdef PERNIX_AVX512_VBMI_ENABLED -int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return mm512_compress_block_avx512vbmi(bit_width, input, scale, output); +namespace { +bool is_valid_block_size(uint32_t block_size) { + return block_size == 64 || block_size == 128 || block_size == 256 || block_size == 512 || block_size == 1024; } - -int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return mm512_compress_block_f64_avx512vbmi(bit_width, input, scale, output); } -int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return mm512_compress_blocks_avx512vbmi(bit_width, input, scale, output, blocks); -} +extern "C" { +pernix_status pernix_compress_block_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + float scale, void* output) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } -int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return mm512_compress_blocks_f64_avx512vbmi(bit_width, input, scale, output, blocks); -} + const auto kernel = pernix::internal::select_compress_block_f32(static_cast(backend), bit_width, block_size); -int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return mm512_decompress_block_avx512vbmi(bit_width, input, scale, output); -} + if (!kernel) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } -int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return mm512_decompress_block_f64_avx512vbmi(bit_width, input, scale, output); + return static_cast(kernel.func(input, scale, output)); } -int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - return mm512_decompress_blocks_avx512vbmi(bit_width, input, scale, output, blocks); -} +pernix_status pernix_compress_blocks_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + float scale, void* output, uint32_t blocks) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } -int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - return mm512_decompress_blocks_f64_avx512vbmi(bit_width, input, scale, output, blocks); -} -#else -int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return mm256_compress_block_avx2(bit_width, input, scale, output); -} + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } -int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return mm256_compress_block_f64_avx2(bit_width, input, scale, output); -} + const auto kernel = pernix::internal::select_compress_blocks_f32(static_cast(backend), bit_width, block_size); -int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return mm256_compress_blocks_avx2(bit_width, input, scale, output, blocks); -} + if (!kernel) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } -int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return mm256_compress_blocks_f64_avx2(bit_width, input, scale, output, blocks); + return static_cast(kernel.func(input, scale, output, blocks)); } -int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return mm256_decompress_block_avx2(bit_width, input, scale, output); -} +pernix_status pernix_decompress_block_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + float scale, void* output, bool sign_values) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } -int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return mm256_decompress_block_f64_avx2(bit_width, input, scale, output); -} + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } -int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - return mm256_decompress_blocks_avx2(bit_width, input, scale, output, blocks); -} + const auto kernel = pernix::internal::select_decompress_block_f32(static_cast(backend), bit_width, block_size, + sign_values); -int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - return mm256_decompress_blocks_f64_avx2(bit_width, input, scale, output, blocks); -} -#endif -#elif defined(PERNIX_BACKEND_ARM64_NEON) -int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return neon_compress_block(bit_width, input, scale, output); -} + if (!kernel) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } -int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return neon_compress_block_f64(bit_width, input, scale, output); + return static_cast(kernel.func(input, scale, output)); } -int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return neon_compress_blocks(bit_width, input, scale, output, blocks); -} +pernix_status pernix_decompress_blocks_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + float scale, void* output, uint32_t blocks, bool sign_values) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } -int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return neon_compress_blocks_f64(bit_width, input, scale, output, blocks); -} + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } -int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return neon_decompress_block(bit_width, input, scale, output); -} + const auto kernel = pernix::internal::select_decompress_blocks_f32(static_cast(backend), bit_width, block_size, + sign_values); -int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return neon_decompress_block_f64(bit_width, input, scale, output); -} + if (!kernel) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } -int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - return neon_decompress_blocks(bit_width, input, scale, output, blocks); + return static_cast(kernel.func(input, scale, output, blocks)); } -int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - return neon_decompress_blocks_f64(bit_width, input, scale, output, blocks); -} -#elif defined(PERNIX_BACKEND_ARM64_SVE) -int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return sve_compress_block(bit_width, input, scale, output); -} +pernix_status pernix_compress_block_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + double scale, void* output) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } -int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return sve_compress_block_f64(bit_width, input, scale, output); -} + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } -int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return sve_compress_blocks(bit_width, input, scale, output, blocks); -} + const auto kernel = pernix::internal::select_compress_block_f64(static_cast(backend), bit_width, block_size); -int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return sve_compress_blocks_f64(bit_width, input, scale, output, blocks); -} + if (!kernel) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } -int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return sve_decompress_block(bit_width, input, scale, output); + return static_cast(kernel.func(input, scale, output)); } -int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return sve_decompress_block_f64(bit_width, input, scale, output); -} +pernix_status pernix_compress_blocks_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + double scale, void* output, uint32_t blocks) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } -int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - return sve_decompress_blocks(bit_width, input, scale, output, blocks); -} + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } -int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - return sve_decompress_blocks_f64(bit_width, input, scale, output, blocks); -} -#elif defined(PERNIX_BACKEND_ARM64_SVE2) -int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return sve2_compress_block(bit_width, input, scale, output); -} - -int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return sve2_compress_block_f64(bit_width, input, scale, output); -} - -int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return sve2_compress_blocks(bit_width, input, scale, output, blocks); -} + const auto kernel = pernix::internal::select_compress_blocks_f64(static_cast(backend), bit_width, block_size); -int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return sve2_compress_blocks_f64(bit_width, input, scale, output, blocks); -} + if (!kernel) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } -int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return sve2_decompress_block(bit_width, input, scale, output); + return static_cast(kernel.func(input, scale, output, blocks)); } -int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return sve2_decompress_block_f64(bit_width, input, scale, output); -} +pernix_status pernix_decompress_block_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + double scale, void* output, bool sign_values) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } -int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - return sve2_decompress_blocks(bit_width, input, scale, output, blocks); -} + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } -int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - return sve2_decompress_blocks_f64(bit_width, input, scale, output, blocks); -} -#else -int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return compress_block_fallback(bit_width, input, scale, output); -} + const auto kernel = pernix::internal::select_decompress_block_f64(static_cast(backend), bit_width, block_size, + sign_values); -int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return compress_block_fallback_f64(bit_width, input, scale, output); -} + if (!kernel) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } -int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return compress_blocks_fallback(bit_width, input, scale, output, blocks); + return static_cast(kernel.func(input, scale, output)); } -int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return compress_blocks_fallback_f64(bit_width, input, scale, output, blocks); -} +pernix_status pernix_decompress_blocks_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, + double scale, void* output, uint32_t blocks, bool sign_values) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } -int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return decompress_block_fallback(bit_width, input, scale, output); -} + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } -int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return decompress_block_fallback_f64(bit_width, input, scale, output); -} + const auto kernel = pernix::internal::select_decompress_blocks_f64(static_cast(backend), bit_width, block_size, + sign_values); -int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - return decompress_blocks_fallback(bit_width, input, scale, output, blocks); -} + if (!kernel) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } -int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - return decompress_blocks_fallback_f64(bit_width, input, scale, output, blocks); + return static_cast(kernel.func(input, scale, output, blocks)); } -#endif - -#ifdef __cplusplus } -} // namespace pernix -#endif // __cplusplus diff --git a/src/x86/avx2/avx2_compression.cpp b/src/x86/avx2/avx2_compression.cpp new file mode 100644 index 0000000..1a7f2c0 --- /dev/null +++ b/src/x86/avx2/avx2_compression.cpp @@ -0,0 +1,193 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ +case N: return Kernel("avx2", &mm256_compress_block_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ +case N: return Kernel("avx2", &mm256_compress_blocks_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ +case N: return Kernel("avx2", &mm256_compress_block_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ +case N: return Kernel("avx2", &mm256_compress_blocks_avx2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: return {"avx2", nullptr}; \ + } + +Kernel select_avx2_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +Kernel select_avx2_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +Kernel select_avx2_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } +} + +Kernel select_avx2_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} diff --git a/src/x86/avx2/avx2_decompression.cpp b/src/x86/avx2/avx2_decompression.cpp new file mode 100644 index 0000000..43558ae --- /dev/null +++ b/src/x86/avx2/avx2_decompression.cpp @@ -0,0 +1,201 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) + + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"avx2", nullptr}; \ + } \ + break + +Kernel select_avx2_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"avx2", nullptr}; + } +} + +Kernel select_avx2_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"avx2", nullptr}; + } +} + +Kernel select_avx2_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"avx2", nullptr}; + } +} + +Kernel select_avx2_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"avx2", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} diff --git a/src/x86/avx2/compression.cpp b/src/x86/avx2/compression.cpp deleted file mode 100644 index 231fc75..0000000 --- a/src/x86/avx2/compression.cpp +++ /dev/null @@ -1,154 +0,0 @@ -#include -#include - -#ifdef PERNIX_AVX2_ENABLED - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_COMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm256_compress_block_avx2(input, scale, output); - -#define PERNIX_COMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm256_compress_blocks_avx2(input, scale, output, blocks); - -int mm256_compress_block_avx2(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_compress_block_f64_avx2(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_compress_blocks_avx2(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm256_compress_blocks_f64_avx2(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED diff --git a/src/x86/avx2/decompression.cpp b/src/x86/avx2/decompression.cpp deleted file mode 100644 index 47c473a..0000000 --- a/src/x86/avx2/decompression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#ifdef PERNIX_AVX2_ENABLED -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm256_decompress_block_avx2(input, scale, output); - -#define PERNIX_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm256_decompress_blocks_avx2(input, scale, output, blocks); - -int mm256_decompress_block_avx2(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_block_f64_avx2(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_blocks_avx2(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_blocks_f64_avx2(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED \ No newline at end of file diff --git a/src/x86/avx512vbmi/avx512vbmi_compression.cpp b/src/x86/avx512vbmi/avx512vbmi_compression.cpp new file mode 100644 index 0000000..f1c36a1 --- /dev/null +++ b/src/x86/avx512vbmi/avx512vbmi_compression.cpp @@ -0,0 +1,193 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ +case N: return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ +case N: return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ +case N: return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ +case N: return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } + +Kernel select_avx512vbmi_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} diff --git a/src/x86/avx512vbmi/avx512vbmi_decompression.cpp b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp new file mode 100644 index 0000000..43f45fc --- /dev/null +++ b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp @@ -0,0 +1,201 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) + + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } \ + break + +Kernel select_avx512vbmi_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"avx512vbmi", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} diff --git a/src/x86/avx512vbmi/compression.cpp b/src/x86/avx512vbmi/compression.cpp deleted file mode 100644 index bdc6635..0000000 --- a/src/x86/avx512vbmi/compression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#if defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_AVX512_VBMI_ENABLED) -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_COMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm512_compress_block_avx512vbmi(input, scale, output); - -#define PERNIX_COMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm512_compress_blocks_avx512vbmi(input, scale, output, blocks); - -int mm512_compress_block_avx512vbmi(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm512_compress_block_f64_avx512vbmi(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm512_compress_blocks_avx512vbmi(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm512_compress_blocks_f64_avx512vbmi(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_AVX512_VBMI_ENABLED) \ No newline at end of file diff --git a/src/x86/avx512vbmi/decompression.cpp b/src/x86/avx512vbmi/decompression.cpp deleted file mode 100644 index 1273f6a..0000000 --- a/src/x86/avx512vbmi/decompression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#if defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_AVX512_VBMI_ENABLED) -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm512_decompress_block_avx512vbmi(input, scale, output); - -#define PERNIX_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm512_decompress_blocks_avx512vbmi(input, scale, output, blocks); - -int mm512_decompress_block_avx512vbmi(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm512_decompress_block_f64_avx512vbmi(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm512_decompress_blocks_avx512vbmi(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm512_decompress_blocks_f64_avx512vbmi(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED && PERNIX_AVX512_VBMI_ENABLED \ No newline at end of file diff --git a/src/x86/bmi2/bmi2_compression.cpp b/src/x86/bmi2/bmi2_compression.cpp new file mode 100644 index 0000000..ea01e04 --- /dev/null +++ b/src/x86/bmi2/bmi2_compression.cpp @@ -0,0 +1,193 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ +case N: return Kernel("bmi2", &mm256_compress_block_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ +case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ +case N: return Kernel("bmi2", &mm256_compress_block_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ +case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: return {"bmi2", nullptr}; \ + } + +Kernel select_bmi2_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} diff --git a/src/x86/bmi2/bmi2_decompression.cpp b/src/x86/bmi2/bmi2_decompression.cpp new file mode 100644 index 0000000..e829c24 --- /dev/null +++ b/src/x86/bmi2/bmi2_decompression.cpp @@ -0,0 +1,201 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) + + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"bmi2", nullptr}; \ + } \ + break + +Kernel select_bmi2_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"bmi2", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} diff --git a/src/x86/bmi2/compression.cpp b/src/x86/bmi2/compression.cpp deleted file mode 100644 index 79c43d8..0000000 --- a/src/x86/bmi2/compression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#if defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_BMI2_ENABLED) -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_COMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm256_compress_block_bmi2(input, scale, output); - -#define PERNIX_COMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm256_compress_blocks_bmi2(input, scale, output, blocks); - -int mm256_compress_block_bmi2(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_compress_block_f64_bmi2(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_compress_blocks_bmi2(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm256_compress_blocks_f64_bmi2(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED && PERNIX_BMI2_ENABLED diff --git a/src/x86/bmi2/decompression.cpp b/src/x86/bmi2/decompression.cpp deleted file mode 100644 index c9cbad7..0000000 --- a/src/x86/bmi2/decompression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#if defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_BMI2_ENABLED) -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm256_decompress_block_bmi2(input, scale, output); - -#define PERNIX_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm256_decompress_blocks_bmi2(input, scale, output, blocks); - -int mm256_decompress_block_bmi2(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_block_f64_bmi2(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_blocks_bmi2(const uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_blocks_f64_bmi2(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED && PERNIX_BMI2_ENABLED \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b5ffe57..62bb90e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,56 +1,7 @@ find_package(PkgConfig) pkg_search_module(GTEST REQUIRED gtest) -include(CheckCXXCompilerFlag) -file(GLOB - PERNIX_ROOT_TEST_SOURCES - CONFIGURE_DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp -) - -file(GLOB_RECURSE - PERNIX_FALLBACK_TEST_SOURCES - CONFIGURE_DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/fallback/*.cpp -) - -set(SOURCE_FILES ${PERNIX_ROOT_TEST_SOURCES} ${PERNIX_FALLBACK_TEST_SOURCES}) - -if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "X86") - file(GLOB_RECURSE - PERNIX_X86_TEST_SOURCES - CONFIGURE_DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/x86/*.cpp - ) - list(APPEND SOURCE_FILES ${PERNIX_X86_TEST_SOURCES}) -elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_NEON") - file(GLOB_RECURSE - PERNIX_ARM64_NEON_TEST_SOURCES - CONFIGURE_DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/arm64/neon/*.cpp - ) - list(APPEND SOURCE_FILES ${PERNIX_ARM64_NEON_TEST_SOURCES}) -elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE") - file(GLOB_RECURSE - PERNIX_ARM64_SVE_TEST_SOURCES - CONFIGURE_DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/arm64/sve/*.cpp - ) - list(APPEND SOURCE_FILES ${PERNIX_ARM64_SVE_TEST_SOURCES}) -elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE2") - file(GLOB_RECURSE - PERNIX_ARM64_SVE2_TEST_SOURCES - CONFIGURE_DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/arm64/sve2/*.cpp - ) - list(APPEND SOURCE_FILES ${PERNIX_ARM64_SVE2_TEST_SOURCES}) -endif () - -file(GLOB_RECURSE - HEADER_FILES - CONFIGURE_DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/include/*.h -) +file(GLOB PERNIX_TEST_SOURCE_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) set(PERNIX_TEST_BLOCK_SIZES 64 128 256 512 CACHE STRING "Block sizes to build as separate test executables") set(PERNIX_TEST_TARGETS) @@ -60,7 +11,7 @@ foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) list(APPEND PERNIX_TEST_TARGETS ${TEST_TARGET}) add_executable(${TEST_TARGET}) - target_sources(${TEST_TARGET} PRIVATE ${SOURCE_FILES} ${HEADER_FILES}) + target_sources(${TEST_TARGET} PRIVATE ${PERNIX_TEST_SOURCE_FILES}) target_link_libraries(${TEST_TARGET} PRIVATE pernix ${GTEST_LDFLAGS}) target_compile_features(${TEST_TARGET} PRIVATE cxx_std_20) target_compile_options(${TEST_TARGET} PRIVATE ${GTEST_CFLAGS}) @@ -70,11 +21,6 @@ endforeach () add_custom_target(pernix_tests DEPENDS ${PERNIX_TEST_TARGETS}) -# check_cxx_compiler_flag(-O3 HAS_OPTIMIZE3_FLAG) -# if (HAS_OPTIMIZE3_FLAG) -# target_compile_options(pernix_tests PRIVATE -O3) -# endif () - include(GoogleTest) foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) gtest_discover_tests(pernix_tests_${BLOCK_SIZE} @@ -88,13 +34,11 @@ endforeach () if (PERNIX_ENABLE_CODE_COVERAGE) message(STATUS "Code coverage enabled for tests") - # set compile and link options for code coverage foreach (TEST_TARGET IN LISTS PERNIX_TEST_TARGETS) target_compile_options(${TEST_TARGET} PRIVATE -g -O0 --coverage) target_link_options(${TEST_TARGET} PRIVATE --coverage) endforeach () - # find lcov and genhtml find_program(LCOV_PROGRAM lcov) find_program(GENHTML_PROGRAM genhtml) if (NOT LCOV_PROGRAM OR NOT GENHTML_PROGRAM) diff --git a/tests/arm64/neon/decompression_tests.cpp b/tests/arm64/neon/decompression_tests.cpp deleted file mode 100644 index 69229be..0000000 --- a/tests/arm64/neon/decompression_tests.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include - -#ifdef PERNIX_BACKEND_ARM64_NEON - -using namespace pernix::arm64::neon; - -TYPED_TEST(DecompressionTest, NeonDecompressBlock) { - std::vector > decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - neon_decompress_block( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, NeonDecompressBlock) { - std::vector > decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - neon_decompress_block( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -#endif \ No newline at end of file diff --git a/tests/arm64/sve/.gitkeep b/tests/arm64/sve/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tests/arm64/sve2/decompression_tests.cpp b/tests/arm64/sve2/decompression_tests.cpp deleted file mode 100644 index 82cb11f..0000000 --- a/tests/arm64/sve2/decompression_tests.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include - -#ifdef PERNIX_BACKEND_ARM64_SVE2 - -using namespace pernix::arm64::sve2; - -TYPED_TEST(DecompressionTest, SVE2DecompressBlock) { - std::vector > decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - sve2_decompress_block( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, SVE2DecompressBlock) { - std::vector > decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - sve2_decompress_block( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -#endif \ No newline at end of file diff --git a/tests/fallback/compression_tests.cpp b/tests/fallback/compression_tests.cpp deleted file mode 100644 index 78249d7..0000000 --- a/tests/fallback/compression_tests.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include -#include - -TYPED_TEST(CompressionTest, FallbackCompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::compress_block_fallback( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectCompressedBlockEqualsReference(*this, compressedData[block], block); - } -} - -TYPED_TEST(CompressionTest64, FallbackCompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::compress_block_fallback( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectCompressedBlockEqualsReference(*this, compressedData[block], block); - } -} diff --git a/tests/fallback/decompression_tests.cpp b/tests/fallback/decompression_tests.cpp deleted file mode 100644 index 08c26c4..0000000 --- a/tests/fallback/decompression_tests.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include - -TYPED_TEST(DecompressionTest, FallbackDecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::decompress_block_fallback( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::decompress_block_fallback( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} diff --git a/tests/fallback/edge_tests.cpp b/tests/fallback/edge_tests.cpp deleted file mode 100644 index 2bdee5a..0000000 --- a/tests/fallback/edge_tests.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include -#include - -#include - -#include -#include -#include -#include - -TEST(FallbackDecompressionEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { - const std::array input{0x08}; - std::array output{}; - - ASSERT_EQ(pernix::decompress_block_fallback<4>(input.data(), 1.0F, output.data()), 0); - - EXPECT_EQ(output[0], -8.0F); -} - -TEST(FallbackCompressionEdgeTest, ClearsUnusedPaddingBytes) { - std::array input{}; - std::array output{}; - output.fill(0xA5); - - ASSERT_EQ(pernix::compress_block_fallback<24>(input.data(), 1.0F, output.data()), 0); - - EXPECT_EQ(output[63], 0); -} - -TEST(FallbackCompressionEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { - std::array input{}; - input[0] = std::numeric_limits::infinity(); - input[1] = -std::numeric_limits::infinity(); - input[2] = std::numeric_limits::quiet_NaN(); - std::array compressed{}; - std::array restored{}; - - ASSERT_EQ(pernix::compress_block_fallback<4>(input.data(), 1.0F, compressed.data()), 0); - ASSERT_EQ(pernix::decompress_block_fallback<4>(compressed.data(), 1.0F, restored.data()), 0); - - EXPECT_EQ(restored[0], 7.0F); - EXPECT_EQ(restored[1], -8.0F); - EXPECT_EQ(restored[2], 0.0F); -} diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp new file mode 100644 index 0000000..5e2b4ef --- /dev/null +++ b/tests/fallback_tests.cpp @@ -0,0 +1,316 @@ +#include + +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Fallback compress: verify byte-exact match against the reference +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, FallbackCompressBlock) { + std::vector> compressed(this->testSet.numberOfBlocks); + + for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + compressed[b].resize(TestFixture::BlockSize); + + const auto status = pernix_compress_block_f32( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getDecompressedData()[b].data(), 1.0f / this->testSet.getScales()[b], + compressed[b].data()); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + expectCompressedBlockEqualsReference(*this, compressed[b], b); + } +} + +TYPED_TEST(CompressionTest64, FallbackCompressBlock) { + std::vector> compressed(this->testSet.numberOfBlocks); + + for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + compressed[b].resize(TestFixture::BlockSize); + + const auto status = pernix_compress_block_f64( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getDecompressedData()[b].data(), 1.0 / this->testSet.getScales()[b], + compressed[b].data()); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + expectCompressedBlockEqualsReference(*this, compressed[b], b); + } +} + +// --------------------------------------------------------------------------- +// Fallback decompress: verify near-source match +// --------------------------------------------------------------------------- + +TYPED_TEST(DecompressionTest, FallbackDecompressBlock) { + std::vector> decompressed(this->testSet.numberOfBlocks); + + for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + decompressed[b].resize(this->testSet.elementsPerBlock); + + const auto status = pernix_decompress_block_f32( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getCompressedData()[b].data(), this->testSet.getScales()[b], + decompressed[b].data(), true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + expectDecompressedBlockNearSource(*this, decompressed[b], b); + } +} + +TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { + std::vector> decompressed(this->testSet.numberOfBlocks); + + for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + decompressed[b].resize(this->testSet.elementsPerBlock); + + const auto status = pernix_decompress_block_f64( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getCompressedData()[b].data(), this->testSet.getScales()[b], + decompressed[b].data(), true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + expectDecompressedBlockNearSource(*this, decompressed[b], b); + } +} + +// --------------------------------------------------------------------------- +// Multi-block roundtrip via compress_blocks / decompress_blocks (fallback) +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, FallbackCompressBlocksRoundtrip) { + const uint32_t nb = this->testSet.numberOfBlocks; + const uint32_t epb = this->testSet.elementsPerBlock; + const uint32_t total = nb * epb; + + std::vector flat(total); + for (uint32_t b = 0; b < nb; b++) { + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, + flat.data() + b * epb); + } + + // Compute a single scale that covers all blocks + float max_abs = 0.0f; + for (uint32_t i = 0; i < total; i++) { + max_abs = std::max(max_abs, std::abs(flat[i])); + } + const float q = static_cast(decltype(this->testSet)::quantization_levels); + const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); + const float scale_inv = 1.0f / scale; + + std::vector compressed(nb * TestFixture::BlockSize); + auto status = pernix_compress_blocks_f32( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + flat.data(), scale_inv, compressed.data(), nb); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + std::vector restored(total); + status = pernix_decompress_blocks_f32( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + compressed.data(), scale, restored.data(), nb, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + const float tol = (std::abs(scale) * 0.5f) + (std::numeric_limits::epsilon() * 16.0f); + for (uint32_t i = 0; i < total; i++) { + EXPECT_NEAR(restored[i], flat[i], tol); + } +} + +TYPED_TEST(CompressionTest64, FallbackCompressBlocksRoundtrip) { + const uint32_t nb = this->testSet.numberOfBlocks; + const uint32_t epb = this->testSet.elementsPerBlock; + const uint32_t total = nb * epb; + + std::vector flat(total); + for (uint32_t b = 0; b < nb; b++) { + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, + flat.data() + b * epb); + } + + // Compute a single scale that covers all blocks + double max_abs = 0.0; + for (uint32_t i = 0; i < total; i++) { + max_abs = std::max(max_abs, std::abs(flat[i])); + } + const double q = static_cast(decltype(this->testSet)::quantization_levels); + const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); + const double scale_inv = 1.0 / scale; + + std::vector compressed(nb * TestFixture::BlockSize); + auto status = pernix_compress_blocks_f64( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + flat.data(), scale_inv, compressed.data(), nb); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + std::vector restored(total); + status = pernix_decompress_blocks_f64( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + compressed.data(), scale, restored.data(), nb, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + const double tol = (std::abs(scale) * 0.5) + (std::numeric_limits::epsilon() * 16.0); + for (uint32_t i = 0; i < total; i++) { + EXPECT_NEAR(restored[i], flat[i], tol); + } +} + +// --------------------------------------------------------------------------- +// blocks API with a single block should match the block API exactly +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, SingleBlockCompressBlocksMatchesBlock) { + const auto& src = this->testSet.getDecompressedData()[0]; + const float scale_inv = 1.0f / this->testSet.getScales()[0]; + + std::vector blockOut(TestFixture::BlockSize); + std::vector blocksOut(TestFixture::BlockSize); + + auto s1 = pernix_compress_block_f32( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + src.data(), scale_inv, blockOut.data()); + ASSERT_EQ(s1, PERNIX_STATUS_OK); + + auto s2 = pernix_compress_blocks_f32( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + src.data(), scale_inv, blocksOut.data(), 1); + ASSERT_EQ(s2, PERNIX_STATUS_OK); + + for (uint32_t i = 0; i < TestFixture::BlockSize; i++) { + EXPECT_EQ(blockOut[i], blocksOut[i]) << "byte " << i; + } +} + +TYPED_TEST(DecompressionTest, SingleBlockDecompressBlocksMatchesBlock) { + const auto& compressed = this->testSet.getCompressedData()[0]; + const float scale = this->testSet.getScales()[0]; + const uint32_t epb = this->testSet.elementsPerBlock; + + std::vector blockOut(epb); + std::vector blocksOut(epb); + + auto s1 = pernix_decompress_block_f32( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + compressed.data(), scale, blockOut.data(), true); + ASSERT_EQ(s1, PERNIX_STATUS_OK); + + auto s2 = pernix_decompress_blocks_f32( + PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + compressed.data(), scale, blocksOut.data(), 1, true); + ASSERT_EQ(s2, PERNIX_STATUS_OK); + + for (uint32_t i = 0; i < epb; i++) { + EXPECT_FLOAT_EQ(blockOut[i], blocksOut[i]) << "element " << i; + } +} + +// --------------------------------------------------------------------------- +// Edge-case behavioural tests (fallback, block_size=64) +// --------------------------------------------------------------------------- + +TEST(FallbackEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { + constexpr uint32_t BS = 64; + const std::array input{0x08}; + + pernix_status st; + std::array output{}; + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, output.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + EXPECT_EQ(output[0], -8.0f); +} + +TEST(FallbackEdgeTest, ClearsUnusedPaddingBytes) { + constexpr uint32_t BS = 64; + constexpr uint32_t BW = 24; + constexpr uint32_t EPB = (BS * 8) / BW; + + std::array input{}; + std::array output{}; + output.fill(0xA5); + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, output.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + EXPECT_EQ(output[BS - 1], 0); +} + +TEST(FallbackEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { + constexpr uint32_t BS = 64; + constexpr uint32_t BW = 4; + constexpr uint32_t EPB = (BS * 8) / BW; + + std::array input{}; + input[0] = std::numeric_limits::infinity(); + input[1] = -std::numeric_limits::infinity(); + input[2] = std::numeric_limits::quiet_NaN(); + + std::array compressed{}; + std::array restored{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), 1.0f, restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(restored[0], 7.0f); + EXPECT_EQ(restored[1], -8.0f); + EXPECT_EQ(restored[2], 0.0f); +} + +// --------------------------------------------------------------------------- +// Error-code contract tests +// --------------------------------------------------------------------------- + +TEST(ErrorCodeTest, UnsupportedBlockSizeReturnsError) { + constexpr uint32_t BS = 32; + float_t src[32] = {}; + uint8_t dst[32] = {}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 8, BS, dst, 1.0f, src, true); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); + + st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, 8, BS, src, 1.0f, dst, 1); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); + + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, 8, BS, dst, 1.0f, src, 1, true); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); +} + +TEST(ErrorCodeTest, UnsupportedBitWidthReturnsError) { + constexpr uint32_t BS = 64; + float_t src[256] = {}; + uint8_t dst[64] = {}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 25, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); +} + +TEST(ErrorCodeTest, NullPointerReturnsError) { + float_t src[64] = {}; + uint8_t dst[64] = {}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, 64, nullptr, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_INVALID_ARGUMENT); + + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, 64, src, 1.0f, nullptr); + EXPECT_EQ(st, PERNIX_STATUS_INVALID_ARGUMENT); +} diff --git a/tests/include/testset.h b/tests/include/testset.h index 5ef4fa1..81acb32 100644 --- a/tests/include/testset.h +++ b/tests/include/testset.h @@ -21,22 +21,14 @@ static_assert(PERNIX_TEST_BLOCK_SIZE % 32 == 0, "PERNIX_TEST_BLOCK_SIZE must be dividable by 32 bytes"); -/** - * A test set for compression and decompression tests. - * It generates random float data, compresses it, and verifies the decompression using the fallback implementation. - * - * @tparam BIT_WIDTH The bit width used for compression (1 to 24). - * @tparam SIGN_VALUES Indicates whether the values are signed or unsigned. - */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && std::is_floating_point_v class TestSet { - // using ValueType = std::conditional_t; using ValueType = uint8_t; using SeedType = std::mt19937::result_type; alignas(64) std::vector > compressedData; - alignas(64) std::vector > decompressedData; + alignas(64) std::vector > sourceData; alignas(64) std::vector scalesData; SeedType seed; @@ -55,14 +47,13 @@ class TestSet { [[nodiscard]] constexpr uint32_t totalElements() const { return numberOfBlocks * elementsPerBlock; } [[nodiscard]] T blockTolerance(const uint32_t block) const { - // Half-step quantization bound + tiny FP slack for rounding edge cases. return (std::abs(scalesData[block]) * static_cast(0.5)) + (std::numeric_limits::epsilon() * static_cast(16)); } explicit TestSet(const uint32_t number_of_blocks, const SeedType initial_seed = testSeed()) : seed(initial_seed), gen(seed), numberOfBlocks(number_of_blocks) { compressedData.resize(numberOfBlocks); - decompressedData.resize(number_of_blocks); + sourceData.resize(number_of_blocks); scalesData.resize(numberOfBlocks); generateData(); @@ -72,7 +63,7 @@ class TestSet { [[nodiscard]] const std::vector >& getCompressedData() const { return compressedData; } - [[nodiscard]] const std::vector >& getDecompressedData() const { return decompressedData; } + [[nodiscard]] const std::vector >& getDecompressedData() const { return sourceData; } [[nodiscard]] SeedType getSeed() const { return seed; } @@ -88,26 +79,29 @@ class TestSet { } private: - // Generate deterministic source data and its fallback-compressed reference. void generateData() { for (uint32_t i = 0; i < numberOfBlocks; i++) { compressedData[i].resize(BLOCK_SIZE); - decompressedData[i].resize(elementsPerBlock); + sourceData[i].resize(elementsPerBlock); for (uint32_t j = 0; j < elementsPerBlock; j++) { - decompressedData[i][j] = dis(gen); + sourceData[i][j] = dis(gen); } - const T b_max = *std::ranges::max_element(decompressedData[i]); - const T b_min = *std::ranges::min_element(decompressedData[i]); + const T b_max = *std::ranges::max_element(sourceData[i]); + const T b_min = *std::ranges::min_element(sourceData[i]); const T b_abs = std::max(std::abs(b_max), std::abs(b_min)); scalesData[i] = (b_abs > static_cast(0) && quantization_levels > static_cast(0)) ? (b_abs / quantization_levels) : std::numeric_limits::epsilon(); - // Compress the data using the fallback implementation - pernix::compress_block_fallback(decompressedData[i].data(), 1 / scalesData[i], - reinterpret_cast(compressedData[i].data())); + if constexpr (std::is_same_v) { + pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, + sourceData[i].data(), 1.0f / scalesData[i], compressedData[i].data()); + } else { + pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, + sourceData[i].data(), 1.0 / scalesData[i], compressedData[i].data()); + } } } }; diff --git a/tests/simd_tests.cpp b/tests/simd_tests.cpp new file mode 100644 index 0000000..6f9bc50 --- /dev/null +++ b/tests/simd_tests.cpp @@ -0,0 +1,188 @@ +#include + +#include +#include + +// --------------------------------------------------------------------------- +// SIMD compress: compress via backend, decompress via fallback, compare source +// --------------------------------------------------------------------------- + +template +void testBackendCompressBlock(FixtureT& fixture, pernix_backend backend) { + using T = std::remove_cvref_t; + + { + std::vector probe(FixtureT::BlockSize); + pernix_status st; + if constexpr (std::is_same_v) { + st = pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getDecompressedData()[0].data(), + 1.0f / fixture.testSet.getScales()[0], probe.data()); + } else { + st = pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getDecompressedData()[0].data(), + 1.0 / fixture.testSet.getScales()[0], probe.data()); + } + if (st != PERNIX_STATUS_OK) { + SUCCEED(); + return; + } + } + + for (uint32_t b = 0; b < fixture.testSet.numberOfBlocks; b++) { + std::vector compressed(FixtureT::BlockSize); + + pernix_status st; + if constexpr (std::is_same_v) { + st = pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getDecompressedData()[b].data(), + 1.0f / fixture.testSet.getScales()[b], compressed.data()); + } else { + st = pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getDecompressedData()[b].data(), + 1.0 / fixture.testSet.getScales()[b], compressed.data()); + } + ASSERT_EQ(st, PERNIX_STATUS_OK); + + std::vector restored(fixture.testSet.elementsPerBlock); + if constexpr (std::is_same_v) { + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, + compressed.data(), fixture.testSet.getScales()[b], restored.data(), true); + } else { + st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, + compressed.data(), fixture.testSet.getScales()[b], restored.data(), true); + } + ASSERT_EQ(st, PERNIX_STATUS_OK); + + expectDecompressedBlockNearSource(fixture, restored, b); + } +} + +// --------------------------------------------------------------------------- +// SIMD decompress: decompress fallback-compressed data via backend, compare source +// --------------------------------------------------------------------------- + +template +void testBackendDecompressBlock(FixtureT& fixture, pernix_backend backend) { + using T = std::remove_cvref_t; + + { + std::vector probe(fixture.testSet.elementsPerBlock); + pernix_status st; + if constexpr (std::is_same_v) { + st = pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getCompressedData()[0].data(), + fixture.testSet.getScales()[0], probe.data(), true); + } else { + st = pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getCompressedData()[0].data(), + fixture.testSet.getScales()[0], probe.data(), true); + } + if (st != PERNIX_STATUS_OK) { + SUCCEED(); + return; + } + } + + for (uint32_t b = 0; b < fixture.testSet.numberOfBlocks; b++) { + std::vector decompressed(fixture.testSet.elementsPerBlock); + + pernix_status st; + if constexpr (std::is_same_v) { + st = pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed.data(), true); + } else { + st = pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed.data(), true); + } + ASSERT_EQ(st, PERNIX_STATUS_OK); + + expectDecompressedBlockNearSource(fixture, decompressed, b); + } +} + +// --------------------------------------------------------------------------- +// x86: AVX2 +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, AVX2CompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_AVX2); +} + +TYPED_TEST(DecompressionTest, AVX2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_AVX2); +} + +TYPED_TEST(CompressionTest64, AVX2CompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_AVX2); +} + +TYPED_TEST(DecompressionTest64, AVX2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_AVX2); +} + +// --------------------------------------------------------------------------- +// x86: BMI2 +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, BMI2CompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_BMI2); +} + +TYPED_TEST(DecompressionTest, BMI2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_BMI2); +} + +TYPED_TEST(CompressionTest64, BMI2CompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_BMI2); +} + +TYPED_TEST(DecompressionTest64, BMI2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_BMI2); +} + +// --------------------------------------------------------------------------- +// x86: AVX512-VBMI +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, AVX512VBMICompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_AVX512_VBMI); +} + +TYPED_TEST(DecompressionTest, AVX512VBMIDecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_AVX512_VBMI); +} + +TYPED_TEST(CompressionTest64, AVX512VBMICompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_AVX512_VBMI); +} + +TYPED_TEST(DecompressionTest64, AVX512VBMIDecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_AVX512_VBMI); +} + +// --------------------------------------------------------------------------- +// ARM64: NEON (decompress only — no compress implementation) +// --------------------------------------------------------------------------- + +TYPED_TEST(DecompressionTest, NeonDecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_ARM64_NEON); +} + +TYPED_TEST(DecompressionTest64, NeonDecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_ARM64_NEON); +} + +// --------------------------------------------------------------------------- +// ARM64: SVE2 (decompress only — no compress implementation) +// --------------------------------------------------------------------------- + +TYPED_TEST(DecompressionTest, SVE2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_ARM64_SVE); +} + +TYPED_TEST(DecompressionTest64, SVE2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_ARM64_SVE); +} diff --git a/tests/x86/avx2/compression_tests.cpp b/tests/x86/avx2/compression_tests.cpp deleted file mode 100644 index bd7f683..0000000 --- a/tests/x86/avx2/compression_tests.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include - -#ifdef PERNIX_AVX2_ENABLED - -TYPED_TEST(CompressionTest, AVX2CompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm256_compress_block_avx2( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -TYPED_TEST(CompressionTest64, AVX2CompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm256_compress_block_avx2( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -#endif // PERNIX_AVX2_ENABLED diff --git a/tests/x86/avx2/decompression_tests.cpp b/tests/x86/avx2/decompression_tests.cpp deleted file mode 100644 index a6fc2c5..0000000 --- a/tests/x86/avx2/decompression_tests.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include - -#ifdef PERNIX_AVX2_ENABLED - -TYPED_TEST(DecompressionTest, AVX2DecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm256_decompress_block_avx2( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, AVX2DecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm256_decompress_block_avx2( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -#endif // PERNIX_AVX2_ENABLED diff --git a/tests/x86/avx512vbmi/compression_tests.cpp b/tests/x86/avx512vbmi/compression_tests.cpp deleted file mode 100644 index a6cb71d..0000000 --- a/tests/x86/avx512vbmi/compression_tests.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include - -#ifdef PERNIX_AVX512_VBMI_ENABLED - -TYPED_TEST(CompressionTest, AVX512VBMICompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm512_compress_block_avx512vbmi( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -TYPED_TEST(CompressionTest64, AVX512VBMICompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm512_compress_block_avx512vbmi( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -#endif // PERNIX_AVX512_VBMI_ENABLED diff --git a/tests/x86/avx512vbmi/decompression_tests.cpp b/tests/x86/avx512vbmi/decompression_tests.cpp deleted file mode 100644 index f44dd8d..0000000 --- a/tests/x86/avx512vbmi/decompression_tests.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include - -#ifdef PERNIX_AVX512_VBMI_ENABLED - -TYPED_TEST(DecompressionTest, AVX512VBMIDecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm512_decompress_block_avx512vbmi( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, AVX512VBMIDecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm512_decompress_block_avx512vbmi( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -#endif // PERNIX_AVX512_VBMI_ENABLED diff --git a/tests/x86/bmi2/compression_tests.cpp b/tests/x86/bmi2/compression_tests.cpp deleted file mode 100644 index b7fc2fd..0000000 --- a/tests/x86/bmi2/compression_tests.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include - -#ifdef PERNIX_BMI2_ENABLED - -TYPED_TEST(CompressionTest, BMI2CompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm256_compress_block_bmi2( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -TYPED_TEST(CompressionTest64, BMI2CompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm256_compress_block_bmi2( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -#endif // PERNIX_BMI2_ENABLED diff --git a/tests/x86/bmi2/decompression_tests.cpp b/tests/x86/bmi2/decompression_tests.cpp deleted file mode 100644 index dd7efc1..0000000 --- a/tests/x86/bmi2/decompression_tests.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include - -#ifdef PERNIX_BMI2_ENABLED - -TYPED_TEST(DecompressionTest, BMI2DecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm256_decompress_block_bmi2( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, BMI2DecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm256_decompress_block_bmi2( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -#endif // PERNIX_BMI2_ENABLED From 8ad45acbbe599183a2f0b44179ee49281b0a40f5 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Mon, 15 Jun 2026 15:29:06 +0200 Subject: [PATCH 14/43] replace number types with pre defined types --- include/pernix/compat.h | 21 + include/pernix/pernix.h | 34 +- include/pernix/pernix.hpp | 127 +- src/arm64/neon/compression.cpp | 40 +- src/arm64/neon/decompression.cpp | 39 +- src/arm64/sve2/compression.cpp | 40 +- src/arm64/sve2/decompression.cpp | 95 +- src/dispatch/cpu_features_arm.cpp | 21 +- src/dispatch/cpu_features_x86.cpp | 8 +- src/dispatch/select.cpp | 820 ++++++------ src/fallback/fallback_compression.cpp | 80 +- src/fallback/fallback_decompression.cpp | 76 +- src/internal/pernix/arm64/neon/common.h | 74 +- src/internal/pernix/arm64/neon/compression.h | 204 +-- src/internal/pernix/arm64/neon/tables.h | 358 ++--- src/internal/pernix/arm64/neon/unpacking.h | 8 +- src/internal/pernix/arm64/sve2/compression.h | 72 +- src/internal/pernix/arm64/sve2/packing.h | 2 +- src/internal/pernix/arm64/sve2/tables.h | 208 ++- src/internal/pernix/arm64/sve2/unpacking.h | 126 +- src/internal/pernix/dispatch/kernel.h | 10 +- src/internal/pernix/dispatch/select.h | 140 +- .../pernix/fallback/avx2_compression.h | 313 ++--- .../pernix/fallback/avx2_decompression.h | 292 ++-- src/internal/pernix/simd_compat.h | 13 - .../pernix/x86/avx2/avx2_compression.h | 794 +++++------ .../pernix/x86/avx2/avx2_decompression.h | 451 ++++--- src/internal/pernix/x86/avx2/avx2_tables.h | 480 +++---- .../x86/avx512vbmi/avx512vbmi_compression.h | 1177 +++++++++-------- .../x86/avx512vbmi/avx512vbmi_decompression.h | 948 ++++++------- src/internal/pernix/x86/avx512vbmi/compat.h | 188 +-- src/internal/pernix/x86/avx512vbmi/packing.h | 42 +- src/internal/pernix/x86/avx512vbmi/tables.h | 640 ++++----- .../pernix/x86/avx512vbmi/unpacking.h | 52 +- .../pernix/x86/bmi2/bmi2_compression.h | 438 +++--- .../pernix/x86/bmi2/bmi2_decompression.h | 492 +++---- src/internal/pernix/x86/utils.h | 14 +- src/pernix.cpp | 70 +- src/x86/avx2/avx2_compression.cpp | 80 +- src/x86/avx2/avx2_decompression.cpp | 76 +- src/x86/avx512vbmi/avx512vbmi_compression.cpp | 80 +- .../avx512vbmi/avx512vbmi_decompression.cpp | 76 +- src/x86/bmi2/bmi2_compression.cpp | 80 +- src/x86/bmi2/bmi2_decompression.cpp | 76 +- tests/fallback_tests.cpp | 136 +- tests/include/testset.h | 146 +- tests/simd_tests.cpp | 56 +- 47 files changed, 5046 insertions(+), 4767 deletions(-) diff --git a/include/pernix/compat.h b/include/pernix/compat.h index 42544ee..498c83a 100644 --- a/include/pernix/compat.h +++ b/include/pernix/compat.h @@ -1,6 +1,10 @@ #ifndef PERNIX_COMPAT_H #define PERNIX_COMPAT_H +#include +#include +#include + #ifndef __always_inline #if defined(__GNUC__) || defined(__clang__) #define __always_inline inline __attribute__((always_inline)) @@ -21,4 +25,21 @@ #define PERNIX_API #endif +// Convenient type declarations +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef uintptr_t uptr; + +typedef int8_t i8; +typedef int16_t i16; +typedef int32_t i32; +typedef int64_t i64; + +typedef float_t f32; +typedef double_t f64; + +typedef size_t usize; + #endif //PERNIX_COMPAT_H diff --git a/include/pernix/pernix.h b/include/pernix/pernix.h index 7a64af6..74d586b 100644 --- a/include/pernix/pernix.h +++ b/include/pernix/pernix.h @@ -2,8 +2,6 @@ #define PERNIX_H #include -#include -#include #if defined(__cplusplus) extern "C" { @@ -27,29 +25,37 @@ typedef enum pernix_backend { PERNIX_BACKEND_ARM64_SVE = 6 } pernix_backend; -PERNIX_API pernix_status pernix_compress_block_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, +PERNIX_API pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, + const void* input, float scale, void* output); -PERNIX_API pernix_status pernix_compress_blocks_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - float scale, void* output, uint32_t blocks); +PERNIX_API pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, + const void* input, + float scale, void* output, u32 blocks); -PERNIX_API pernix_status pernix_decompress_block_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, +PERNIX_API pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, + const void* input, float scale, void* output, bool sign_values); -PERNIX_API pernix_status pernix_decompress_blocks_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - float scale, void* output, uint32_t blocks, bool sign_values); +PERNIX_API pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, + const void* input, + float scale, void* output, u32 blocks, bool sign_values); -PERNIX_API pernix_status pernix_compress_block_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, +PERNIX_API pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, + const void* input, double scale, void* output); -PERNIX_API pernix_status pernix_compress_blocks_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - double scale, void* output, uint32_t blocks); +PERNIX_API pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, + const void* input, + double scale, void* output, u32 blocks); -PERNIX_API pernix_status pernix_decompress_block_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, +PERNIX_API pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, + const void* input, double scale, void* output, bool sign_values); -PERNIX_API pernix_status pernix_decompress_blocks_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - double scale, void* output, uint32_t blocks, bool sign_values); +PERNIX_API pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, + const void* input, + double scale, void* output, u32 blocks, bool sign_values); #if defined(__cplusplus) } diff --git a/include/pernix/pernix.hpp b/include/pernix/pernix.hpp index 67301bb..d197f72 100644 --- a/include/pernix/pernix.hpp +++ b/include/pernix/pernix.hpp @@ -1,5 +1,7 @@ #ifndef PERNIX_HPP #define PERNIX_HPP + +#include #include namespace pernix { @@ -13,139 +15,150 @@ enum class Backend { Arm64Sve = PERNIX_BACKEND_ARM64_SVE }; -__always_inline int compress_block(Backend backend, const uint8_t bit_width, const uint32_t block_size, - const std::span input, const float scale, std::span output) { - return pernix_compress_block_f32(static_cast(backend), bit_width, block_size, input.data(), scale, output.data()); +__always_inline int compress_block(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const float scale, std::span output) { + return pernix_compress_block_f32(static_cast(backend), bit_width, block_size, input.data(), + scale, output.data()); } -__always_inline int compress_block(Backend backend, const uint8_t bit_width, const uint32_t block_size, - const std::span input, const double scale, std::span output) { - return pernix_compress_block_f64(static_cast(backend), bit_width, block_size, input.data(), scale, output.data()); +__always_inline int compress_block(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const double scale, std::span output) { + return pernix_compress_block_f64(static_cast(backend), bit_width, block_size, input.data(), + scale, output.data()); } -__always_inline int decompress_block(Backend backend, const uint8_t bit_width, const uint32_t block_size, - const std::span input, const float scale, std::span output, +__always_inline int decompress_block(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const float scale, std::span output, const bool sign_values = true) { - return pernix_decompress_block_f32(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), + return pernix_decompress_block_f32(static_cast(backend), bit_width, block_size, input.data(), + scale, output.data(), sign_values); } -__always_inline int decompress_block(Backend backend, const uint8_t bit_width, const uint32_t block_size, - const std::span input, const double scale, std::span output, +__always_inline int decompress_block(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const double scale, std::span output, const bool sign_values = true) { - return pernix_decompress_block_f64(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), + return pernix_decompress_block_f64(static_cast(backend), bit_width, block_size, input.data(), + scale, output.data(), sign_values); } -__always_inline int compress_blocks(Backend backend, const uint8_t bit_width, const uint32_t block_size, - const std::span input, const float scale, std::span output, - const uint32_t blocks) { - return pernix_compress_blocks_f32(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), +__always_inline int compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const float scale, std::span output, + const u32 blocks) { + return pernix_compress_blocks_f32(static_cast(backend), bit_width, block_size, input.data(), + scale, output.data(), blocks); } -__always_inline int compress_blocks(Backend backend, const uint8_t bit_width, const uint32_t block_size, - const std::span input, const double scale, std::span output, - const uint32_t blocks) { - return pernix_compress_blocks_f64(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), +__always_inline int compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const double scale, std::span output, + const u32 blocks) { + return pernix_compress_blocks_f64(static_cast(backend), bit_width, block_size, input.data(), + scale, output.data(), blocks); } -__always_inline int decompress_blocks(Backend backend, const uint8_t bit_width, const uint32_t block_size, - const std::span input, const float scale, std::span output, - const uint32_t blocks, const bool sign_values = true) { - return pernix_decompress_blocks_f32(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), +__always_inline int decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const float scale, std::span output, + const u32 blocks, const bool sign_values = true) { + return pernix_decompress_blocks_f32(static_cast(backend), bit_width, block_size, input.data(), + scale, output.data(), blocks, sign_values); } -__always_inline int decompress_blocks(Backend backend, const uint8_t bit_width, const uint32_t block_size, - const std::span input, const double scale, std::span output, - const uint32_t blocks, const bool sign_values = true) { - return pernix_decompress_blocks_f64(static_cast(backend), bit_width, block_size, input.data(), scale, output.data(), +__always_inline int decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const double scale, std::span output, + const u32 blocks, const bool sign_values = true) { + return pernix_decompress_blocks_f64(static_cast(backend), bit_width, block_size, input.data(), + scale, output.data(), blocks, sign_values); } // convenience overloads without backend (defaults to Auto) -__always_inline int compress_block(const uint8_t bit_width, const uint32_t block_size, const std::span input, - const float scale, const std::span output) { +__always_inline int compress_block(const u8 bit_width, const u32 block_size, const std::span input, + const float scale, const std::span output) { return compress_block(Backend::Auto, bit_width, block_size, input, scale, output); } -__always_inline int compress_block(const uint8_t bit_width, const uint32_t block_size, const std::span input, - const double scale, const std::span output) { +__always_inline int compress_block(const u8 bit_width, const u32 block_size, const std::span input, + const double scale, const std::span output) { return compress_block(Backend::Auto, bit_width, block_size, input, scale, output); } -__always_inline int decompress_block(const uint8_t bit_width, const uint32_t block_size, const std::span input, +__always_inline int decompress_block(const u8 bit_width, const u32 block_size, const std::span input, const float scale, const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); } -__always_inline int decompress_block(const uint8_t bit_width, const uint32_t block_size, const std::span input, - const double scale, const std::span output, const bool sign_values = true) { +__always_inline int decompress_block(const u8 bit_width, const u32 block_size, const std::span input, + const double scale, const std::span output, + const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); } -__always_inline int compress_blocks(const uint8_t bit_width, const uint32_t block_size, const std::span input, - const float scale, const std::span output, const uint32_t blocks) { +__always_inline int compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const float scale, const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks); } -__always_inline int compress_blocks(const uint8_t bit_width, const uint32_t block_size, const std::span input, - const double scale, const std::span output, const uint32_t blocks) { +__always_inline int compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const double scale, const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks); } -__always_inline int decompress_blocks(const uint8_t bit_width, const uint32_t block_size, const std::span input, - const float scale, const std::span output, const uint32_t blocks, +__always_inline int decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const float scale, const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); } -__always_inline int decompress_blocks(const uint8_t bit_width, const uint32_t block_size, const std::span input, - const double scale, const std::span output, const uint32_t blocks, +__always_inline int decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const double scale, const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); } // convenience overloads without backend and without block_size (defaults to 64) -__always_inline int compress_block(const uint8_t bit_width, const std::span input, const float scale, - const std::span output) { +__always_inline int compress_block(const u8 bit_width, const std::span input, const float scale, + const std::span output) { return compress_block(Backend::Auto, bit_width, 64, input, scale, output); } -__always_inline int compress_block(const uint8_t bit_width, const std::span input, const double scale, - const std::span output) { +__always_inline int compress_block(const u8 bit_width, const std::span input, const double scale, + const std::span output) { return compress_block(Backend::Auto, bit_width, 64, input, scale, output); } -__always_inline int decompress_block(const uint8_t bit_width, const std::span input, const float scale, +__always_inline int decompress_block(const u8 bit_width, const std::span input, const float scale, const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); } -__always_inline int decompress_block(const uint8_t bit_width, const std::span input, const double scale, +__always_inline int decompress_block(const u8 bit_width, const std::span input, const double scale, const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); } -__always_inline int compress_blocks(const uint8_t bit_width, const std::span input, const float scale, - const std::span output, const uint32_t blocks) { +__always_inline int compress_blocks(const u8 bit_width, const std::span input, const float scale, + const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks); } -__always_inline int compress_blocks(const uint8_t bit_width, const std::span input, const double scale, - const std::span output, const uint32_t blocks) { +__always_inline int compress_blocks(const u8 bit_width, const std::span input, const double scale, + const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks); } -__always_inline int decompress_blocks(const uint8_t bit_width, const std::span input, const float scale, - const std::span output, const uint32_t blocks, const bool sign_values = true) { +__always_inline int decompress_blocks(const u8 bit_width, const std::span input, const float scale, + const std::span output, const u32 blocks, + const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); } -__always_inline int decompress_blocks(const uint8_t bit_width, const std::span input, const double scale, - const std::span output, const uint32_t blocks, const bool sign_values = true) { +__always_inline int decompress_blocks(const u8 bit_width, const std::span input, const double scale, + const std::span output, const u32 blocks, + const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); } } diff --git a/src/arm64/neon/compression.cpp b/src/arm64/neon/compression.cpp index 81c1cc2..3594303 100644 --- a/src/arm64/neon/compression.cpp +++ b/src/arm64/neon/compression.cpp @@ -2,27 +2,27 @@ #include namespace pernix::internal { -Kernel select_neon_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { - (void)bit_width; - (void)block_size; - return {"neon", nullptr}; -} + Kernel select_neon_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"neon", nullptr}; + } -Kernel select_neon_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { - (void)bit_width; - (void)block_size; - return {"neon", nullptr}; -} + Kernel select_neon_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"neon", nullptr}; + } -Kernel select_neon_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { - (void)bit_width; - (void)block_size; - return {"neon", nullptr}; -} + Kernel select_neon_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"neon", nullptr}; + } -Kernel select_neon_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { - (void)bit_width; - (void)block_size; - return {"neon", nullptr}; -} + Kernel select_neon_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"neon", nullptr}; + } } diff --git a/src/arm64/neon/decompression.cpp b/src/arm64/neon/decompression.cpp index 94da2fb..3083926 100644 --- a/src/arm64/neon/decompression.cpp +++ b/src/arm64/neon/decompression.cpp @@ -1,26 +1,29 @@ #include #include +using pernix::arm64::neon::neon_decompress_block; +using pernix::arm64::neon::neon_decompress_blocks; + namespace pernix::internal { #define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ case N: \ - if (sign_values) return Kernel("neon", &arm64::neon::neon_decompress_block); \ - return Kernel("neon", &arm64::neon::neon_decompress_block) + if (sign_values) return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) #define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ case N: \ - if (sign_values) return Kernel("neon", &arm64::neon::neon_decompress_blocks); \ - return Kernel("neon", &arm64::neon::neon_decompress_blocks) + if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) #define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ case N: \ - if (sign_values) return Kernel("neon", &arm64::neon::neon_decompress_block); \ - return Kernel("neon", &arm64::neon::neon_decompress_block) + if (sign_values) return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) #define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ case N: \ - if (sign_values) return Kernel("neon", &arm64::neon::neon_decompress_blocks); \ - return Kernel("neon", &arm64::neon::neon_decompress_blocks) + if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ case BS: \ @@ -146,47 +149,51 @@ case N: \ } \ break -Kernel select_neon_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { +Kernel select_neon_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { switch (block_size) { PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"neon", nullptr}; + default: + return {"neon", nullptr}; } } -Kernel select_neon_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { +Kernel select_neon_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { switch (block_size) { PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"neon", nullptr}; + default: + return {"neon", nullptr}; } } -Kernel select_neon_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { +Kernel select_neon_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { switch (block_size) { PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"neon", nullptr}; + default: + return {"neon", nullptr}; } } -Kernel select_neon_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { +Kernel select_neon_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { switch (block_size) { PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"neon", nullptr}; + default: + return {"neon", nullptr}; } } diff --git a/src/arm64/sve2/compression.cpp b/src/arm64/sve2/compression.cpp index c6d8dd0..1839f12 100644 --- a/src/arm64/sve2/compression.cpp +++ b/src/arm64/sve2/compression.cpp @@ -2,27 +2,27 @@ #include namespace pernix::internal { -Kernel select_sve2_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { - (void)bit_width; - (void)block_size; - return {"sve2", nullptr}; -} + Kernel select_sve2_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"sve2", nullptr}; + } -Kernel select_sve2_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { - (void)bit_width; - (void)block_size; - return {"sve2", nullptr}; -} + Kernel select_sve2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"sve2", nullptr}; + } -Kernel select_sve2_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { - (void)bit_width; - (void)block_size; - return {"sve2", nullptr}; -} + Kernel select_sve2_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"sve2", nullptr}; + } -Kernel select_sve2_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { - (void)bit_width; - (void)block_size; - return {"sve2", nullptr}; -} + Kernel select_sve2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"sve2", nullptr}; + } } diff --git a/src/arm64/sve2/decompression.cpp b/src/arm64/sve2/decompression.cpp index ba796e0..8e65f6e 100644 --- a/src/arm64/sve2/decompression.cpp +++ b/src/arm64/sve2/decompression.cpp @@ -1,26 +1,29 @@ #include #include +using pernix::arm64::sve2::sve2_decompress_block; +using pernix::arm64::sve2::sve2_decompress_blocks; + namespace pernix::internal { #define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ case N: \ - if (sign_values) return Kernel("sve2", &arm64::sve2::sve2_decompress_block); \ - return Kernel("sve2", &arm64::sve2::sve2_decompress_block) + if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) #define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ case N: \ - if (sign_values) return Kernel("sve2", &arm64::sve2::sve2_decompress_blocks); \ - return Kernel("sve2", &arm64::sve2::sve2_decompress_blocks) + if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) #define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ case N: \ - if (sign_values) return Kernel("sve2", &arm64::sve2::sve2_decompress_block); \ - return Kernel("sve2", &arm64::sve2::sve2_decompress_block) + if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) #define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ case N: \ - if (sign_values) return Kernel("sve2", &arm64::sve2::sve2_decompress_blocks); \ - return Kernel("sve2", &arm64::sve2::sve2_decompress_blocks) + if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ case BS: \ @@ -146,49 +149,53 @@ case N: \ } \ break -Kernel select_sve2_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"sve2", nullptr}; + Kernel select_sve2_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"sve2", nullptr}; + } } -} -Kernel select_sve2_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"sve2", nullptr}; + Kernel select_sve2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"sve2", nullptr}; + } } -} -Kernel select_sve2_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"sve2", nullptr}; + Kernel select_sve2_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"sve2", nullptr}; + } } -} -Kernel select_sve2_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"sve2", nullptr}; + Kernel select_sve2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"sve2", nullptr}; + } } -} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 diff --git a/src/dispatch/cpu_features_arm.cpp b/src/dispatch/cpu_features_arm.cpp index 1b4ac78..525962b 100644 --- a/src/dispatch/cpu_features_arm.cpp +++ b/src/dispatch/cpu_features_arm.cpp @@ -1,5 +1,15 @@ #include +#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) +#include +#ifndef HWCAP_SVE +#define HWCAP_SVE (1 << 22) +#endif +#ifndef HWCAP2_SVE2 +#define HWCAP2_SVE2 (1 << 1) +#endif +#endif + namespace pernix::internal { CpuFeatures detect_cpu_features() { CpuFeatures features{}; @@ -11,11 +21,12 @@ CpuFeatures detect_cpu_features() { features.neon = true; #endif - // sve -#if defined(__aarch64__) || defined(_M_ARM64) -#ifdef __ARM_FEATURE_SVE - features.sve = true; -#endif + // sve / sve2 — runtime detection via getauxval on Linux +#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) + unsigned long hwcap = getauxval(AT_HWCAP); + unsigned long hwcap2 = getauxval(AT_HWCAP2); + features.sve = (hwcap & HWCAP_SVE) != 0; + features.sve2 = (hwcap2 & HWCAP2_SVE2) != 0; #endif return features; diff --git a/src/dispatch/cpu_features_x86.cpp b/src/dispatch/cpu_features_x86.cpp index f43245c..973df4d 100644 --- a/src/dispatch/cpu_features_x86.cpp +++ b/src/dispatch/cpu_features_x86.cpp @@ -1,6 +1,6 @@ #include -#include +#include #if defined(__x86_64__) || defined(_M_X64) || defined(__i386) || defined(_M_IX86) #if defined(_MSC_VER) @@ -19,7 +19,7 @@ void cpuid(int out[4], int leaf, int subleaf) { __cpuidex(out, leaf, subleaf); } -std::uint64_t xgetbv(unsigned int index) { +u64 xgetbv(unsigned int index) { return _xgetbv(index); } @@ -29,7 +29,7 @@ void cpuid(int out[4], int leaf, int subleaf) { __cpuid_count(leaf, subleaf, out[0], out[1], out[2], out[3]); } -std::uint64_t xgetbv(unsigned int index) { +u64 xgetbv(unsigned int index) { return _xgetbv(index); } @@ -54,7 +54,7 @@ CpuFeatures detect_cpu_features() { return features; } - const std::uint64_t xcr0 = xgetbv(0); + const u64 xcr0 = xgetbv(0); const bool xmm_enabled = (xcr0 & 0x2) != 0; const bool ymm_enabled = (xcr0 & 0x4) != 0; diff --git a/src/dispatch/select.cpp b/src/dispatch/select.cpp index b619af1..ca4673e 100644 --- a/src/dispatch/select.cpp +++ b/src/dispatch/select.cpp @@ -2,683 +2,719 @@ #include namespace pernix::internal { -Kernel select_compress_block_f32(Backend backend, uint8_t bit_width, uint32_t block_size) { - switch (backend) { - case Backend::Auto: - return select_auto_compress_block_f32(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_block_f32(bit_width, block_size); + Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size) { + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f32(bit_width, block_size); #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_block_f32(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_block_f32(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_block_f32(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_block_f32(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return select_sve2_compress_block_f32(bit_width, block_size); + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f32(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } #endif - default: - return {"invalid_backend", nullptr}; + default: + return {"invalid_backend", nullptr}; + } } -} -Kernel select_compress_blocks_f32(Backend backend, uint8_t bit_width, uint32_t block_size) { - switch (backend) { - case Backend::Auto: - return select_auto_compress_blocks_f32(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_blocks_f32(bit_width, block_size); + Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size) { + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f32(bit_width, block_size); #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_blocks_f32(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_blocks_f32(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_blocks_f32(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return select_sve2_compress_blocks_f32(bit_width, block_size); + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f32(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } #endif - default: - return {"invalid_backend", nullptr}; + default: + return {"invalid_backend", nullptr}; + } } -} -Kernel select_compress_block_f64(Backend backend, uint8_t bit_width, uint32_t block_size) { - switch (backend) { - case Backend::Auto: - return select_auto_compress_block_f64(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_block_f64(bit_width, block_size); + Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size) { + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f64(bit_width, block_size); #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_block_f64(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_block_f64(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_block_f64(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_block_f64(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return select_sve2_compress_block_f64(bit_width, block_size); + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f64(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } #endif - default: - return {"invalid_backend", nullptr}; + default: + return {"invalid_backend", nullptr}; + } } -} -Kernel select_compress_blocks_f64(Backend backend, uint8_t bit_width, uint32_t block_size) { - switch (backend) { - case Backend::Auto: - return select_auto_compress_blocks_f64(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_blocks_f64(bit_width, block_size); + Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size) { + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f64(bit_width, block_size); #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_blocks_f64(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_blocks_f64(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_blocks_f64(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return select_sve2_compress_blocks_f64(bit_width, block_size); + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f64(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } #endif - default: - return {"invalid_backend", nullptr}; + default: + return {"invalid_backend", nullptr}; + } } -} -Kernel select_decompress_block_f32(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values) { - switch (backend) { - case Backend::Auto: - return select_auto_decompress_block_f32(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); + Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, + bool sign_values) { + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } #endif - default: - return {"invalid_backend", nullptr}; + default: + return {"invalid_backend", nullptr}; + } } -} -Kernel select_decompress_blocks_f32(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values) { - switch (backend) { - case Backend::Auto: - return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); + Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, + bool sign_values) { + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } #endif - default: - return {"invalid_backend", nullptr}; + default: + return {"invalid_backend", nullptr}; + } } -} -Kernel select_decompress_block_f64(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values) { - switch (backend) { - case Backend::Auto: - return select_auto_decompress_block_f64(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); + Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, + bool sign_values) { + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } #endif - default: - return {"invalid_backend", nullptr}; + default: + return {"invalid_backend", nullptr}; + } } -} -Kernel select_decompress_blocks_f64(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values) { - switch (backend) { - case Backend::Auto: - return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); + Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, + bool sign_values) { + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } #endif - default: - return {"invalid_backend", nullptr}; + default: + return {"invalid_backend", nullptr}; + } } -} -Kernel select_auto_compress_block_f32(uint8_t bit_width, uint32_t block_size) { + Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size) { #if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { - return kernel; + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { - return kernel; + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { - return kernel; + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { - return kernel; + if (features.neon) { + if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve) { - if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { - return kernel; + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { + return kernel; + } } - } #endif - return select_fallback_compress_block_f32(bit_width, block_size); -} + return select_fallback_compress_block_f32(bit_width, block_size); + } -Kernel select_auto_compress_blocks_f32(uint8_t bit_width, uint32_t block_size) { + Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size) { #if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { - return kernel; + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { - return kernel; + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { - return kernel; + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { - return kernel; + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve) { - if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { - return kernel; + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } } - } #endif - return select_fallback_compress_blocks_f32(bit_width, block_size); -} + return select_fallback_compress_blocks_f32(bit_width, block_size); + } -Kernel select_auto_compress_block_f64(uint8_t bit_width, uint32_t block_size) { + Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size) { #if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { - return kernel; + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { - return kernel; + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { - return kernel; + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { - return kernel; + if (features.neon) { + if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve) { - if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { - return kernel; + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { + return kernel; + } } - } #endif - return select_fallback_compress_block_f64(bit_width, block_size); -} + return select_fallback_compress_block_f64(bit_width, block_size); + } -Kernel select_auto_compress_blocks_f64(uint8_t bit_width, uint32_t block_size) { + Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size) { #if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { - return kernel; + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { - return kernel; + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { - return kernel; + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { - return kernel; + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve) { - if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { - return kernel; + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } } - } #endif - return select_fallback_compress_blocks_f64(bit_width, block_size); -} + return select_fallback_compress_blocks_f64(bit_width, block_size); + } -Kernel select_auto_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values) { + Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values) { #if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve) { - if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif - return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); -} + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); + } -Kernel select_auto_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values) { + Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values) { #if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve) { - if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif - return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); -} + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); + } -Kernel select_auto_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values) { + Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values) { #if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve) { - if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif - return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); -} + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); + } -Kernel select_auto_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values) { + Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values) { #if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve) { - if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } } - } #endif - return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); -} + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); + } } diff --git a/src/fallback/fallback_compression.cpp b/src/fallback/fallback_compression.cpp index eeaa34d..4f1e955 100644 --- a/src/fallback/fallback_compression.cpp +++ b/src/fallback/fallback_compression.cpp @@ -135,53 +135,53 @@ case N: return Kernel("fallback", &compress_blocks_fallback default: return {"fallback", nullptr}; \ } -Kernel select_fallback_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"fallback", nullptr}; + Kernel select_fallback_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } } -} -Kernel select_fallback_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"fallback", nullptr}; + Kernel select_fallback_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } } -} -Kernel select_fallback_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"fallback", nullptr}; + Kernel select_fallback_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } } -} -Kernel select_fallback_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"fallback", nullptr}; + Kernel select_fallback_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } } -} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 diff --git a/src/fallback/fallback_decompression.cpp b/src/fallback/fallback_decompression.cpp index 232b831..184397f 100644 --- a/src/fallback/fallback_decompression.cpp +++ b/src/fallback/fallback_decompression.cpp @@ -146,49 +146,53 @@ case N: \ } \ break -Kernel select_fallback_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"fallback", nullptr}; + Kernel select_fallback_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"fallback", nullptr}; + } } -} -Kernel select_fallback_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"fallback", nullptr}; + Kernel select_fallback_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"fallback", nullptr}; + } } -} -Kernel select_fallback_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"fallback", nullptr}; + Kernel select_fallback_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"fallback", nullptr}; + } } -} -Kernel select_fallback_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"fallback", nullptr}; + Kernel select_fallback_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"fallback", nullptr}; + } } -} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 diff --git a/src/internal/pernix/arm64/neon/common.h b/src/internal/pernix/arm64/neon/common.h index 2677416..6cd9ad0 100644 --- a/src/internal/pernix/arm64/neon/common.h +++ b/src/internal/pernix/arm64/neon/common.h @@ -10,9 +10,9 @@ namespace pernix::arm64::neon::internal { float64x2_t val[8]; }; - static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t remaining_elements) { - const uint32_t tail_bits = remaining_elements * bit_width; - const uint32_t tail_bytes = (tail_bits + 7u) / 8u; + static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements) { + const u32 tail_bits = remaining_elements * bit_width; + const u32 tail_bytes = (tail_bits + 7u) / 8u; return tail_bytes; } @@ -110,110 +110,110 @@ __always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t &input }; } -__always_inline uint8x16_t neon_load_tail_elements_int8(const uint8_t *input, const uint32_t tail_bytes_count) { - uint8_t buffer[16] = {0}; +__always_inline uint8x16_t neon_load_tail_elements_int8(const u8 *input, const u32 tail_bytes_count) { + u8 buffer[16] = {0}; std::memcpy(buffer, input, tail_bytes_count); return vld1q_u8(buffer); } -__always_inline uint16x8_t neon_load_tail_elements_int16(const uint8_t *input, const uint32_t tail_bytes_count) { - uint16_t buffer[8] = {0}; +__always_inline uint16x8_t neon_load_tail_elements_int16(const u8 *input, const u32 tail_bytes_count) { + u16 buffer[8] = {0}; std::memcpy(buffer, input, tail_bytes_count); return vld1q_u16(buffer); } -__always_inline uint32x4_t neon_load_tail_elements_int32(const uint8_t *input, const uint32_t tail_bytes_count) { - uint32_t buffer[4] = {0}; +__always_inline uint32x4_t neon_load_tail_elements_int32(const u8 *input, const u32 tail_bytes_count) { + u32 buffer[4] = {0}; std::memcpy(buffer, input, tail_bytes_count); return vld1q_u32(buffer); } -__always_inline float32x4_t neon_load_tail_elements_f32(const uint8_t *input, const uint32_t tail_elements) { +__always_inline float32x4_t neon_load_tail_elements_f32(const u8 *input, const u32 tail_elements) { float32_t buffer[4] = {0.0f}; std::memcpy(buffer, input, tail_elements * sizeof(float32_t)); return vld1q_f32(buffer); } -__always_inline float64x2_t neon_load_tail_elements_f64(const uint8_t *input, const uint32_t tail_elements) { +__always_inline float64x2_t neon_load_tail_elements_f64(const u8 *input, const u32 tail_elements) { float64_t buffer[2] = {0.0}; std::memcpy(buffer, input, tail_elements * sizeof(float64_t)); return vld1q_f64(buffer); } -__always_inline void neon_store_tail_elements_int8(uint8_t *output, const uint8x16x4_t &data, - const uint32_t tail_elements) { - uint8_t buffer[16 * 4]; - for (uint32_t i = 0; i < 4; ++i) { +__always_inline void neon_store_tail_elements_int8(u8 *output, const uint8x16x4_t &data, + const u32 tail_elements) { + u8 buffer[16 * 4]; + for (u32 i = 0; i < 4; ++i) { vst1q_u8(buffer + i * 16, data.val[i]); } - std::memcpy(output, buffer, tail_elements * sizeof(uint8_t)); + std::memcpy(output, buffer, tail_elements * sizeof(u8)); } -__always_inline void neon_store_tail_elements_int16(uint16_t *output, const uint16x8x4_t &data, - const uint32_t tail_elements) { - uint16_t buffer[8 * 4]; - for (uint32_t i = 0; i < 4; ++i) { +__always_inline void neon_store_tail_elements_int16(u16 *output, const uint16x8x4_t &data, + const u32 tail_elements) { + u16 buffer[8 * 4]; + for (u32 i = 0; i < 4; ++i) { vst1q_u16(buffer + i * 8, data.val[i]); } - std::memcpy(output, buffer, tail_elements * sizeof(uint16_t)); + std::memcpy(output, buffer, tail_elements * sizeof(u16)); } -__always_inline void neon_store_tail_elements_int32(uint32_t *output, const uint32x4x4_t &data, - const uint32_t tail_elements) { - uint32_t buffer[4 * 4]; - for (uint32_t i = 0; i < 4; ++i) { +__always_inline void neon_store_tail_elements_int32(u32 *output, const uint32x4x4_t &data, + const u32 tail_elements) { + u32 buffer[4 * 4]; + for (u32 i = 0; i < 4; ++i) { vst1q_u32(buffer + i * 4, data.val[i]); } - std::memcpy(output, buffer, tail_elements * sizeof(uint32_t)); + std::memcpy(output, buffer, tail_elements * sizeof(u32)); } __always_inline void neon_store_tail_elements_f32(float32_t *output, const float32x4x4_t &data, - const uint32_t tail_elements) { + const u32 tail_elements) { float32_t buffer[16 * 4]; - for (uint32_t i = 0; i < 4; ++i) { + for (u32 i = 0; i < 4; ++i) { vst1q_f32(buffer + i * 4, data.val[i]); } std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); } __always_inline void neon_store_tail_elements_f32(float32_t *output, const float32x4x2_t &data, - const uint32_t tail_elements) { + const u32 tail_elements) { float32_t buffer[8 * 2]; - for (uint32_t i = 0; i < 2; ++i) { + for (u32 i = 0; i < 2; ++i) { vst1q_f32(buffer + i * 4, data.val[i]); } std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); } __always_inline void neon_store_tail_elements_f32(float32_t *output, const float32x4_t &data, - const uint32_t tail_elements) { + const u32 tail_elements) { float32_t buffer[4]; vst1q_f32(buffer, data); std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); } __always_inline void neon_store_tail_elements_f64(float64_t *output, const float64x2x4_t &data, - const uint32_t tail_elements) { + const u32 tail_elements) { float64_t buffer[2 * 4]; - for (uint32_t i = 0; i < 4; ++i) { + for (u32 i = 0; i < 4; ++i) { vst1q_f64(buffer + i * 2, data.val[i]); } std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); } __always_inline void neon_store_tail_elements_f64(float64_t *output, const float64x2x2_t &data, - const uint32_t tail_elements) { + const u32 tail_elements) { float64_t buffer[2 * 2]; - for (uint32_t i = 0; i < 2; ++i) { + for (u32 i = 0; i < 2; ++i) { vst1q_f64(buffer + i * 2, data.val[i]); } std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); } __always_inline void neon_store_tail_elements_f64(float64_t *output, const float64x2x8_t &data, - const uint32_t tail_elements) { + const u32 tail_elements) { float64_t buffer[2 * 8]; - for (uint32_t i = 0; i < 8; ++i) { + for (u32 i = 0; i < 8; ++i) { vst1q_f64(buffer + i * 2, data.val[i]); } std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); diff --git a/src/internal/pernix/arm64/neon/compression.h b/src/internal/pernix/arm64/neon/compression.h index f88fbb6..83d46c3 100644 --- a/src/internal/pernix/arm64/neon/compression.h +++ b/src/internal/pernix/arm64/neon/compression.h @@ -8,114 +8,114 @@ #include namespace pernix::arm64::neon { -namespace internal { -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; -} -} // namespace internal - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::neon_compress_block_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::neon_compress_block_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::neon_compress_block_17to24(input, scale, output); + namespace internal { + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_1to8(const u8 * __restrict__ input, const f32 scale, + f32 * __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; + } + + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_9to16(const u8 * __restrict__ input, const f32 scale, + f32 * __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; + } + + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_17to24(const u8 * __restrict__ input, const f32 scale, + f32 * __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; + } + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_1to8(const u8 * __restrict__ input, const f64 scale, + f64 * __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; + } + + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_9to16(const u8 * __restrict__ input, const f64 scale, + f64 * __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; + } + + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_17to24(const u8 * __restrict__ input, const f64 scale, + f64 * __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; + } + } // namespace internal + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block(const u8 * __restrict__ input, const f32 scale, + f32 * __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_compress_block_17to24(input, scale, output); + } + return 0; } - return 0; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::neon_compress_block_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::neon_compress_block_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::neon_compress_block_17to24(input, scale, output); - } - return 0; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_compress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - neon_compress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block(const u8 * __restrict__ input, const f64 scale, + f64 * __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_compress_block_17to24(input, scale, output); + } + return 0; } - return 0; -} + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int neon_compress_blocks(const u8 * __restrict__ input, const f32 scale, f32 * __restrict__ output, + const u32 blocks) { + const u8 *block_input = input; + f32 *block_output = output; -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_compress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; + for (u32 block = 0; block < blocks; ++block) { + neon_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; + } - for (uint32_t block = 0; block < blocks; ++block) { - neon_compress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int neon_compress_blocks(const u8 * __restrict__ input, const f64 scale, f64 * __restrict__ output, + const u32 blocks) { + const u8 *block_input = input; + f64 *block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + neon_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + return 0; } - return 0; -} } // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_COMPRESSION_H diff --git a/src/internal/pernix/arm64/neon/tables.h b/src/internal/pernix/arm64/neon/tables.h index d085551..beb7c47 100644 --- a/src/internal/pernix/arm64/neon/tables.h +++ b/src/internal/pernix/arm64/neon/tables.h @@ -7,206 +7,208 @@ #include namespace pernix::arm64::neon::internal { -namespace detail { -inline constexpr std::size_t neon_vector_width = 128; -inline constexpr uint8_t inactive_lane = 0xff; - -template -constexpr bool table_indices_are_valid(const std::array& table) { - return std::ranges::all_of(table, [](const uint8_t index) { - return index == inactive_lane || index < Elements; - }); -} - -template -constexpr std::array make_primary_permute() { - static_assert(LANE_BITS % 8 == 0); - - constexpr std::size_t lane_bytes = LANE_BITS / 8; - static_assert(ELEMENTS % lane_bytes == 0); - - std::array table{}; - table.fill(inactive_lane); - - for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t first_byte = bit_start / 8; - const std::size_t base = entry * lane_bytes; - - for (std::size_t lane_byte = 0; lane_byte < lane_bytes; ++lane_byte) { - table[base + lane_byte] = static_cast(first_byte + lane_byte); + namespace detail { + inline constexpr std::size_t neon_vector_width = 128; + inline constexpr u8 inactive_lane = 0xff; + + template + constexpr bool table_indices_are_valid(const std::array &table) { + return std::ranges::all_of(table, [](const u8 index) { + return index == inactive_lane || index < Elements; + }); } - } - return table; -} + template + constexpr std::array make_primary_permute() { + static_assert(LANE_BITS % 8 == 0); -template -constexpr std::array make_spill_permute() { - static_assert(LANE_BITS % 8 == 0); + constexpr std::size_t lane_bytes = LANE_BITS / 8; + static_assert(ELEMENTS % lane_bytes == 0); - constexpr std::size_t lane_bytes = LANE_BITS / 8; - static_assert(ELEMENTS % lane_bytes == 0); + std::array table{}; + table.fill(inactive_lane); - std::array table{}; - table.fill(inactive_lane); + for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t first_byte = bit_start / 8; + const std::size_t base = entry * lane_bytes; - for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t first_byte = bit_start / 8; - const std::size_t bit_offset = bit_start % 8; - const std::size_t base = entry * lane_bytes; + for (std::size_t lane_byte = 0; lane_byte < lane_bytes; ++lane_byte) { + table[base + lane_byte] = static_cast(first_byte + lane_byte); + } + } - if (bit_offset + BIT_WIDTH > LANE_BITS) { - table[base] = static_cast(first_byte + lane_bytes); + return table; } - } - return table; -} + template + constexpr std::array make_spill_permute() { + static_assert(LANE_BITS % 8 == 0); -template -constexpr std::array make_shift_right() { - std::array table{}; - table.fill(0); + constexpr std::size_t lane_bytes = LANE_BITS / 8; + static_assert(ELEMENTS % lane_bytes == 0); - for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t bit_offset = bit_start % 8u; + std::array table{}; + table.fill(inactive_lane); - table[entry] = -static_cast(bit_offset); - } + for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t first_byte = bit_start / 8; + const std::size_t bit_offset = bit_start % 8; + const std::size_t base = entry * lane_bytes; - return table; -} + if (bit_offset + BIT_WIDTH > LANE_BITS) { + table[base] = static_cast(first_byte + lane_bytes); + } + } -template -constexpr std::array make_shift_left_for_spill() { - std::array table{}; - table.fill(0); + return table; + } + + template + constexpr std::array make_shift_right() { + std::array table{}; + table.fill(0); + + for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t bit_offset = bit_start % 8u; + + table[entry] = -static_cast(bit_offset); + } + + return table; + } + + template + constexpr std::array make_shift_left_for_spill() { + std::array table{}; + table.fill(0); - for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t bit_offset = bit_start % 8u; - const bool spills = bit_offset + BIT_WIDTH > LANE_BITS; + for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t bit_offset = bit_start % 8u; + const bool spills = bit_offset + BIT_WIDTH > LANE_BITS; - table[entry] = spills ? static_cast(LANE_BITS - bit_offset) : 0; - } + table[entry] = spills ? static_cast(LANE_BITS - bit_offset) : 0; + } - return table; -} + return table; + } + + template + constexpr std::array make_contiguous_permute_32() { + static_assert(ELEMENTS % 4 == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (std::size_t entry = 0; entry < ELEMENTS / 4; ++entry) { + const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + const std::size_t bit_end = bit_start + BIT_WIDTH - 1; + const std::size_t first_byte = bit_start / 8; + const std::size_t last_byte = bit_end / 8; + const std::size_t base = entry * 4; + + for (std::size_t byte = first_byte; byte <= last_byte; ++byte) { + table[base + (byte - first_byte)] = static_cast(byte); + } + } + + return table; + } -template -constexpr std::array make_contiguous_permute_32() { - static_assert(ELEMENTS % 4 == 0); + template + constexpr std::array make_shift_right_32() { + std::array table{}; + table.fill(0); - std::array table{}; - table.fill(inactive_lane); + for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { + const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; - for (std::size_t entry = 0; entry < ELEMENTS / 4; ++entry) { - const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; - const std::size_t bit_end = bit_start + BIT_WIDTH - 1; - const std::size_t first_byte = bit_start / 8; - const std::size_t last_byte = bit_end / 8; - const std::size_t base = entry * 4; + table[entry] = -static_cast(bit_start % 8u); + } - for (std::size_t byte = first_byte; byte <= last_byte; ++byte) { - table[base + (byte - first_byte)] = static_cast(byte); + return table; } - } - - return table; -} - -template -constexpr std::array make_shift_right_32() { - std::array table{}; - table.fill(0); - - for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { - const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; - - table[entry] = -static_cast(bit_start % 8u); - } - - return table; -} -} // namespace detail - -template -struct table_unpacking; - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && VECTOR_WIDTH == detail::neon_vector_width) -struct table_unpacking { -private: - static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; - static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 8; - -public: - static constexpr uint8_t bit_width = BIT_WIDTH; - - alignas(64) static constexpr std::array permute1 = - detail::make_primary_permute(); - alignas(64) static constexpr std::array permute2 = - detail::make_spill_permute(); - alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); - alignas(64) static constexpr std::array shift2 = - detail::make_shift_left_for_spill(); - - static_assert(PERMUTE_ELEMENTS == 16); - static_assert(SHIFT_ELEMENTS == 16); - static_assert(detail::table_indices_are_valid(permute1)); - static_assert(detail::table_indices_are_valid(permute2)); -}; - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && VECTOR_WIDTH == detail::neon_vector_width) -struct table_unpacking { -private: - static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; - static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 16; - -public: - static constexpr uint8_t bit_width = BIT_WIDTH; - - alignas(64) static constexpr std::array permute1 = - detail::make_primary_permute(); - alignas(64) static constexpr std::array permute2 = - detail::make_spill_permute(); - alignas(64) static constexpr std::array shift1 = - detail::make_shift_right(); - alignas(64) static constexpr std::array shift2 = - detail::make_shift_left_for_spill(); - - static_assert(PERMUTE_ELEMENTS == 16); - static_assert(SHIFT_ELEMENTS == 8); - static_assert(detail::table_indices_are_valid(permute1)); - static_assert(detail::table_indices_are_valid(permute2)); -}; - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && VECTOR_WIDTH == detail::neon_vector_width && START_BIT_OFFSET < 8) -struct table_unpacking { -private: - static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; - static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 32; - -public: - static constexpr uint8_t bit_width = BIT_WIDTH; - - alignas(64) static constexpr std::array permute = - detail::make_contiguous_permute_32(); - alignas(64) static constexpr std::array shift = - detail::make_shift_right_32(); - - static_assert(PERMUTE_ELEMENTS == 16); - static_assert(SHIFT_ELEMENTS == 4); - static_assert(detail::table_indices_are_valid(permute)); -}; - -template -struct table_packing; + } // namespace detail + + template + struct table_unpacking; + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && VECTOR_WIDTH == detail::neon_vector_width) + struct table_unpacking { + private: + static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 8; + + public: + static constexpr u8 bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute1 = + detail::make_primary_permute(); + alignas(64) static constexpr std::array permute2 = + detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); + alignas(64) static constexpr std::array shift2 = + detail::make_shift_left_for_spill(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 16); + static_assert(detail::table_indices_are_valid(permute1)); + static_assert(detail::table_indices_are_valid(permute2)); + }; + + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && VECTOR_WIDTH == detail::neon_vector_width) + struct table_unpacking { + private: + static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 16; + + public: + static constexpr u8 bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute1 = + detail::make_primary_permute(); + alignas(64) static constexpr std::array permute2 = + detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = + detail::make_shift_right(); + alignas(64) static constexpr std::array shift2 = + detail::make_shift_left_for_spill(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 8); + static_assert(detail::table_indices_are_valid(permute1)); + static_assert(detail::table_indices_are_valid(permute2)); + }; + + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && VECTOR_WIDTH == detail::neon_vector_width && START_BIT_OFFSET < + 8) + struct table_unpacking { + private: + static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 32; + + public: + static constexpr u8 bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute = + detail::make_contiguous_permute_32(); + alignas(64) static constexpr std::array shift = + detail::make_shift_right_32(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 4); + static_assert(detail::table_indices_are_valid(permute)); + }; + + template + struct table_packing; } // namespace pernix::arm64::internal #endif // PERNIX_ARM64_NEON_TABLES_H diff --git a/src/internal/pernix/arm64/neon/unpacking.h b/src/internal/pernix/arm64/neon/unpacking.h index e70fbbc..cc9bcf0 100644 --- a/src/internal/pernix/arm64/neon/unpacking.h +++ b/src/internal/pernix/arm64/neon/unpacking.h @@ -7,7 +7,7 @@ using namespace pernix::arm64::neon::internal; namespace pernix::arm64::neon::internal::b128 { - template + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t &input) { if constexpr (BIT_WIDTH == 8) { @@ -43,7 +43,7 @@ __always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t &input) { } } - template + template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t &input) { if constexpr (BIT_WIDTH == 16) { @@ -77,7 +77,7 @@ __always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t &input) { } } - template + template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline int32x4_t neon_unpack_epi32_17to24(const uint32x4_t &input) { using tables = table_unpacking; @@ -92,7 +92,7 @@ __always_inline int32x4_t neon_unpack_epi32_17to24(const uint32x4_t &input) { constexpr int sign_shift = 32 - BIT_WIDTH; return vshrq_n_s32(vreinterpretq_s32_u32(vshlq_n_u32(value, sign_shift)), sign_shift); } else { - constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1u; + constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1u; return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); } } diff --git a/src/internal/pernix/arm64/sve2/compression.h b/src/internal/pernix/arm64/sve2/compression.h index 72d9229..33f48fe 100644 --- a/src/internal/pernix/arm64/sve2/compression.h +++ b/src/internal/pernix/arm64/sve2/compression.h @@ -7,42 +7,42 @@ #include namespace pernix { -namespace internal { -template -inline constexpr bool sve2_compression_unimplemented_v = false; -} // namespace internal - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_compress_block(const float_t*, float_t, uint8_t*) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); - return -1; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_compress_block(const double_t*, double_t, uint8_t*) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); - return -1; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_compress_blocks(const float_t*, float_t, uint8_t*, uint32_t) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); - return -1; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_compress_blocks(const double_t*, double_t, uint8_t*, uint32_t) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); - return -1; -} + namespace internal { + template + inline constexpr bool sve2_compression_unimplemented_v = false; + } // namespace internal + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int sve2_compress_block(const f32 *, f32, u8 *) { + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); + return -1; + } + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int sve2_compress_block(const f64 *, f64, u8 *) { + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); + return -1; + } + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int sve2_compress_blocks(const f32 *, f32, u8 *, u32) { + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); + return -1; + } + + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int sve2_compress_blocks(const f64 *, f64, u8 *, u32) { + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); + return -1; + } } // namespace pernix #endif // PERNIX_ARM64_SVE2_COMPRESSION_H diff --git a/src/internal/pernix/arm64/sve2/packing.h b/src/internal/pernix/arm64/sve2/packing.h index 5cf2355..7d644f2 100644 --- a/src/internal/pernix/arm64/sve2/packing.h +++ b/src/internal/pernix/arm64/sve2/packing.h @@ -4,7 +4,7 @@ #include namespace pernix::arm64::sve2::internal { - template + template inline constexpr bool packing_unimplemented_v = false; } // namespace pernix::arm64::sve2::internal diff --git a/src/internal/pernix/arm64/sve2/tables.h b/src/internal/pernix/arm64/sve2/tables.h index 8ef61d8..a11fcde 100644 --- a/src/internal/pernix/arm64/sve2/tables.h +++ b/src/internal/pernix/arm64/sve2/tables.h @@ -3,117 +3,115 @@ #include -#include - namespace pernix::arm64::sve2::internal { - template - struct table_unpacking { - static constexpr uint8_t bit_width = BIT_WIDTH; - - static svbool_t pg_b8() { return svptrue_b8(); } - - static svbool_t pg_b16() { return svptrue_b16(); } - - static svbool_t pg_b32() { return svptrue_b32(); } - }; - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) - struct table_unpacking { - static constexpr uint8_t bit_width = BIT_WIDTH; - - static svuint8_t permute() { - const svbool_t pg = svptrue_b8(); - return svlsr_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 3); - } - - static svuint8_t spill_permute() { - const svbool_t pg = svptrue_b8(); - return svadd_n_u8_x(pg, permute(), 1); - } - - static svuint8_t shift() { - const svbool_t pg = svptrue_b8(); - return svand_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 7); +template +struct table_unpacking { + static constexpr u8 bit_width = BIT_WIDTH; + + static svbool_t pg_b8() { return svptrue_b8(); } + + static svbool_t pg_b16() { return svptrue_b16(); } + + static svbool_t pg_b32() { return svptrue_b32(); } +}; + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +struct table_unpacking { + static constexpr u8 bit_width = BIT_WIDTH; + + static svuint8_t permute() { + const svbool_t pg = svptrue_b8(); + return svlsr_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 3); + } + + static svuint8_t spill_permute() { + const svbool_t pg = svptrue_b8(); + return svadd_n_u8_x(pg, permute(), 1); + } + + static svuint8_t shift() { + const svbool_t pg = svptrue_b8(); + return svand_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 7); + } + + static svuint8_t spill_shift() { + const svbool_t pg = svptrue_b8(); + return svsub_u8_x(pg, svdup_n_u8(8), shift()); + } +}; + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +struct table_unpacking { + static constexpr u8 bit_width = BIT_WIDTH; + + static svuint8_t permute() { + const svbool_t pg = svptrue_b8(); + const svuint8_t lane = svindex_u8(0, 1); + const svuint8_t elem = svlsr_n_u8_x(pg, lane, 1); + const svuint8_t byte = svand_n_u8_x(pg, lane, 1); + + svuint8_t first; + if constexpr (BIT_WIDTH == 16) { + first = svlsl_n_u8_x(pg, elem, 1); + } else { + constexpr u8 extra_bits = BIT_WIDTH - 8u; + const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); + const svuint8_t low = svlsr_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), 3); + first = svadd_u8_x(pg, elem, svadd_u8_x(pg, high, low)); } - static svuint8_t spill_shift() { - const svbool_t pg = svptrue_b8(); - return svsub_u8_x(pg, svdup_n_u8(8), shift()); - } - }; - - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) - struct table_unpacking { - static constexpr uint8_t bit_width = BIT_WIDTH; - - static svuint8_t permute() { - const svbool_t pg = svptrue_b8(); - const svuint8_t lane = svindex_u8(0, 1); - const svuint8_t elem = svlsr_n_u8_x(pg, lane, 1); - const svuint8_t byte = svand_n_u8_x(pg, lane, 1); - - svuint8_t first; - if constexpr (BIT_WIDTH == 16) { - first = svlsl_n_u8_x(pg, elem, 1); - } else { - constexpr uint8_t extra_bits = BIT_WIDTH - 8u; - const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); - const svuint8_t low = svlsr_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), 3); - first = svadd_u8_x(pg, elem, svadd_u8_x(pg, high, low)); - } - - return svadd_u8_x(pg, first, byte); + return svadd_u8_x(pg, first, byte); + } + + static svuint8_t spill_permute() { + const svbool_t pg = svptrue_b8(); + return svadd_n_u8_x(pg, permute(), 2); + } + + static svuint16_t shift() { + const svbool_t pg = svptrue_b16(); + return svand_n_u16_x(pg, svmul_n_u16_x(pg, svindex_u16(0, 1), BIT_WIDTH), 7); + } + + static svuint16_t spill_shift() { + const svbool_t pg = svptrue_b16(); + const svuint16_t bit_shift = shift(); + const svuint16_t spill = svsub_u16_x(pg, svdup_n_u16(16), bit_shift); + return svsel_u16(svcmpgt_n_u16(pg, bit_shift, 16u - BIT_WIDTH), spill, svdup_n_u16(16)); + } +}; + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && START_BIT_OFFSET < 8) +struct table_unpacking { + static constexpr u8 bit_width = BIT_WIDTH; + + static svuint8_t permute() { + const svbool_t pg = svptrue_b8(); + const svuint8_t lane = svindex_u8(0, 1); + const svuint8_t elem = svlsr_n_u8_x(pg, lane, 2); + const svuint8_t byte = svand_n_u8_x(pg, lane, 3); + + svuint8_t first = svmul_n_u8_x(pg, elem, BIT_WIDTH / 8u); + if constexpr (BIT_WIDTH % 8u != 0) { + constexpr u8 extra_bits = BIT_WIDTH % 8u; + const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); + const svuint8_t low_bits = + svadd_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), START_BIT_OFFSET); + first = svadd_u8_x(pg, first, svadd_u8_x(pg, high, svlsr_n_u8_x(pg, low_bits, 3))); } - static svuint8_t spill_permute() { - const svbool_t pg = svptrue_b8(); - return svadd_n_u8_x(pg, permute(), 2); - } - - static svuint16_t shift() { - const svbool_t pg = svptrue_b16(); - return svand_n_u16_x(pg, svmul_n_u16_x(pg, svindex_u16(0, 1), BIT_WIDTH), 7); - } + return svadd_u8_x(pg, first, byte); + } - static svuint16_t spill_shift() { - const svbool_t pg = svptrue_b16(); - const svuint16_t bit_shift = shift(); - const svuint16_t spill = svsub_u16_x(pg, svdup_n_u16(16), bit_shift); - return svsel_u16(svcmpgt_n_u16(pg, bit_shift, 16u - BIT_WIDTH), spill, svdup_n_u16(16)); - } - }; - - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && START_BIT_OFFSET < 8) - struct table_unpacking { - static constexpr uint8_t bit_width = BIT_WIDTH; - - static svuint8_t permute() { - const svbool_t pg = svptrue_b8(); - const svuint8_t lane = svindex_u8(0, 1); - const svuint8_t elem = svlsr_n_u8_x(pg, lane, 2); - const svuint8_t byte = svand_n_u8_x(pg, lane, 3); - - svuint8_t first = svmul_n_u8_x(pg, elem, BIT_WIDTH / 8u); - if constexpr (BIT_WIDTH % 8u != 0) { - constexpr uint8_t extra_bits = BIT_WIDTH % 8u; - const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); - const svuint8_t low_bits = - svadd_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), START_BIT_OFFSET); - first = svadd_u8_x(pg, first, svadd_u8_x(pg, high, svlsr_n_u8_x(pg, low_bits, 3))); - } - - return svadd_u8_x(pg, first, byte); - } - - static svuint32_t shift() { - const svbool_t pg = svptrue_b32(); - return svand_n_u32_x(pg, svadd_n_u32_x(pg, svmul_n_u32_x(pg, svindex_u32(0, 1), BIT_WIDTH), - START_BIT_OFFSET), 7); - } - }; + static svuint32_t shift() { + const svbool_t pg = svptrue_b32(); + return svand_n_u32_x(pg, svadd_n_u32_x(pg, svmul_n_u32_x(pg, svindex_u32(0, 1), BIT_WIDTH), + START_BIT_OFFSET), 7); + } +}; } // namespace pernix::arm64::sve2::internal #endif // PERNIX_ARM64_SVE2_TABLES_H diff --git a/src/internal/pernix/arm64/sve2/unpacking.h b/src/internal/pernix/arm64/sve2/unpacking.h index 8bb7e3c..3d0825a 100644 --- a/src/internal/pernix/arm64/sve2/unpacking.h +++ b/src/internal/pernix/arm64/sve2/unpacking.h @@ -6,89 +6,89 @@ #include "tables.h" namespace pernix::arm64::sve2::internal { - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline svint8_t sve2_unpack_epi8_1to8(const svuint8_t input, const svuint8_t permute, const svuint8_t shift, const svuint8_t spill_permute, const svuint8_t spill_shift) { - if constexpr (BIT_WIDTH == 8) { - return svreinterpret_s8(input); - } else { - const svbool_t pg = svptrue_b8(); + if constexpr (BIT_WIDTH == 8) { + return svreinterpret_s8(input); + } else { + const svbool_t pg = svptrue_b8(); + + const svuint8_t permuted = svtbl_u8(input, permute); + svuint8_t unpacked = svlsr_u8_x(pg, permuted, shift); + + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + const svuint8_t spill_permuted_values = svtbl_u8(input, spill_permute); + const svuint8_t spill_shifted = svlsl_u8_x(pg, spill_permuted_values, spill_shift); + unpacked = svorr_u8_x(pg, unpacked, spill_shifted); + } - const svuint8_t permuted = svtbl_u8(input, permute); - svuint8_t unpacked = svlsr_u8_x(pg, permuted, shift); + if constexpr (BIT_WIDTH == 1) { + unpacked = svand_n_u8_x(pg, unpacked, 1); + return svreinterpret_s8(unpacked); + } else { + constexpr int sign_shift = 8 - BIT_WIDTH; - if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { - const svuint8_t spill_permuted_values = svtbl_u8(input, spill_permute); - const svuint8_t spill_shifted = svlsl_u8_x(pg, spill_permuted_values, spill_shift); - unpacked = svorr_u8_x(pg, unpacked, spill_shifted); - } + unpacked = svlsl_n_u8_x(pg, unpacked, sign_shift); - if constexpr (BIT_WIDTH == 1) { - unpacked = svand_n_u8_x(pg, unpacked, 1); - return svreinterpret_s8(unpacked); + if constexpr (SIGN_VALUES) { + return svasr_n_s8_x(pg, svreinterpret_s8_u8(unpacked), sign_shift); } else { - constexpr int sign_shift = 8 - BIT_WIDTH; - - unpacked = svlsl_n_u8_x(pg, unpacked, sign_shift); - - if constexpr (SIGN_VALUES) { - return svasr_n_s8_x(pg, svreinterpret_s8_u8(unpacked), sign_shift); - } else { - return svreinterpret_s8_u8(svlsr_n_u8_x(pg, unpacked, sign_shift)); - } + return svreinterpret_s8_u8(svlsr_n_u8_x(pg, unpacked, sign_shift)); } } } +} - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline svint16_t sve2_unpack_epi16_9to16(const svuint16_t input, const svuint8_t permute, const svuint16_t shift, const svuint8_t spill_permute, const svuint16_t spill_shift) { - if constexpr (BIT_WIDTH == 16) { - return svreinterpret_s16(input); - } else { - const svbool_t pg = svptrue_b16(); - - const svuint8_t permuted = svtbl_u8(svreinterpret_u8_u16(input), permute); - svuint16_t shifted = svlsr_u16_x(pg, svreinterpret_u16_u8(permuted), shift); - - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const svuint8_t spill_permuted_values = svtbl_u8(svreinterpret_u8_u16(input), spill_permute); - const svuint16_t spill_shifted = svlsl_u16_x(pg, svreinterpret_u16_u8(spill_permuted_values), - spill_shift); - shifted = svorr_u16_x(pg, shifted, spill_shifted); - } - - constexpr int sign_shift = 16 - BIT_WIDTH; - shifted = svlsl_n_u16_x(pg, shifted, sign_shift); - - if constexpr (SIGN_VALUES) { - return svasr_n_s16_x(pg, svreinterpret_s16_u16(shifted), sign_shift); - } else { - return svreinterpret_s16_u16(svlsr_n_u16_x(pg, shifted, sign_shift)); - } + if constexpr (BIT_WIDTH == 16) { + return svreinterpret_s16(input); + } else { + const svbool_t pg = svptrue_b16(); + + const svuint8_t permuted = svtbl_u8(svreinterpret_u8_u16(input), permute); + svuint16_t shifted = svlsr_u16_x(pg, svreinterpret_u16_u8(permuted), shift); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const svuint8_t spill_permuted_values = svtbl_u8(svreinterpret_u8_u16(input), spill_permute); + const svuint16_t spill_shifted = svlsl_u16_x(pg, svreinterpret_u16_u8(spill_permuted_values), + spill_shift); + shifted = svorr_u16_x(pg, shifted, spill_shifted); } - } - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline svint32_t sve2_unpack_epi32_17to24(const svuint8_t input) { - using table = table_unpacking; - - const svbool_t pg = svptrue_b32(); - const svuint8_t permuted = svtbl_u8(input, table::permute()); - const svuint32_t unpacked = svlsr_u32_x(pg, svreinterpret_u32_u8(permuted), table::shift()); + constexpr int sign_shift = 16 - BIT_WIDTH; + shifted = svlsl_n_u16_x(pg, shifted, sign_shift); if constexpr (SIGN_VALUES) { - constexpr int sign_shift = 32 - BIT_WIDTH; - return svasr_n_s32_x(pg, svreinterpret_s32_u32(svlsl_n_u32_x(pg, unpacked, sign_shift)), sign_shift); + return svasr_n_s16_x(pg, svreinterpret_s16_u16(shifted), sign_shift); } else { - constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1u; - return svreinterpret_s32_u32(svand_n_u32_x(pg, unpacked, mask)); + return svreinterpret_s16_u16(svlsr_n_u16_x(pg, shifted, sign_shift)); } } +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline svint32_t sve2_unpack_epi32_17to24(const svuint8_t input) { + using table = table_unpacking; + + const svbool_t pg = svptrue_b32(); + const svuint8_t permuted = svtbl_u8(input, table::permute()); + const svuint32_t unpacked = svlsr_u32_x(pg, svreinterpret_u32_u8(permuted), table::shift()); + + if constexpr (SIGN_VALUES) { + constexpr int sign_shift = 32 - BIT_WIDTH; + return svasr_n_s32_x(pg, svreinterpret_s32_u32(svlsl_n_u32_x(pg, unpacked, sign_shift)), sign_shift); + } else { + constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1u; + return svreinterpret_s32_u32(svand_n_u32_x(pg, unpacked, mask)); + } +} } // namespace pernix::arm64::sve2::internal #endif // PERNIX_ARM64_SVE2_UNPACKING_H diff --git a/src/internal/pernix/dispatch/kernel.h b/src/internal/pernix/dispatch/kernel.h index 1d51c64..b7c0618 100644 --- a/src/internal/pernix/dispatch/kernel.h +++ b/src/internal/pernix/dispatch/kernel.h @@ -1,14 +1,14 @@ #ifndef PERNIX_KERNEL_H #define PERNIX_KERNEL_H -#include +#include #include namespace pernix::internal { -using KernelBlockF32Func = int (*)(const void*, float, void*); -using KernelBlocksF32Func = int (*)(const void*, float, void*, unsigned int); -using KernelBlockF64Func = int (*)(const void*, double, void*); -using KernelBlocksF64Func = int (*)(const void*, double, void*, unsigned int); +using KernelBlockF32Func = i32 (*)(const void*, f32, void*); +using KernelBlocksF32Func = i32 (*)(const void*, f32, void*, u32); +using KernelBlockF64Func = i32 (*)(const void*, f64, void*); +using KernelBlocksF64Func = i32 (*)(const void*, f64, void*, u32); template struct Kernel { diff --git a/src/internal/pernix/dispatch/select.h b/src/internal/pernix/dispatch/select.h index 152117a..d00357c 100644 --- a/src/internal/pernix/dispatch/select.h +++ b/src/internal/pernix/dispatch/select.h @@ -5,153 +5,165 @@ #include namespace pernix::internal { -Kernel select_compress_block_f32(Backend backend, uint8_t bit_width, uint32_t block_size); + Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size); -Kernel select_compress_blocks_f32(Backend backend, uint8_t bit_width, uint32_t block_size); + Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size); -Kernel select_compress_block_f64(Backend backend, uint8_t bit_width, uint32_t block_size); + Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size); -Kernel select_compress_blocks_f64(Backend backend, uint8_t bit_width, uint32_t block_size); + Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size); -Kernel select_decompress_block_f32(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, + bool sign_values); -Kernel select_decompress_blocks_f32(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, + bool sign_values); -Kernel select_decompress_block_f64(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, + bool sign_values); -Kernel select_decompress_blocks_f64(Backend backend, uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, + bool sign_values); -Kernel select_auto_compress_block_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size); -Kernel select_auto_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size); -Kernel select_auto_compress_block_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size); -Kernel select_auto_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size); -Kernel select_auto_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_auto_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_auto_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_auto_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_fallback_compress_block_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_fallback_compress_block_f32(u8 bit_width, u32 block_size); -Kernel select_fallback_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_fallback_compress_blocks_f32(u8 bit_width, u32 block_size); -Kernel select_fallback_compress_block_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_fallback_compress_block_f64(u8 bit_width, u32 block_size); -Kernel select_fallback_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_fallback_compress_blocks_f64(u8 bit_width, u32 block_size); -Kernel select_fallback_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel + select_fallback_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_fallback_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_fallback_decompress_blocks_f32(u8 bit_width, u32 block_size, + bool sign_values); -Kernel select_fallback_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel + select_fallback_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_fallback_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_fallback_decompress_blocks_f64(u8 bit_width, u32 block_size, + bool sign_values); #if defined(PERNIX_BUILD_X86_AVX2) -Kernel select_avx2_compress_block_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_avx2_compress_block_f32(u8 bit_width, u32 block_size); -Kernel select_avx2_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_avx2_compress_blocks_f32(u8 bit_width, u32 block_size); -Kernel select_avx2_compress_block_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_avx2_compress_block_f64(u8 bit_width, u32 block_size); -Kernel select_avx2_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_avx2_compress_blocks_f64(u8 bit_width, u32 block_size); -Kernel select_avx2_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_avx2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_avx2_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_avx2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_avx2_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_avx2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_avx2_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_avx2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) -Kernel select_bmi2_compress_block_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_bmi2_compress_block_f32(u8 bit_width, u32 block_size); -Kernel select_bmi2_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_bmi2_compress_blocks_f32(u8 bit_width, u32 block_size); -Kernel select_bmi2_compress_block_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_bmi2_compress_block_f64(u8 bit_width, u32 block_size); -Kernel select_bmi2_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_bmi2_compress_blocks_f64(u8 bit_width, u32 block_size); -Kernel select_bmi2_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_bmi2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_bmi2_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_bmi2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_bmi2_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_bmi2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_bmi2_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_bmi2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) -Kernel select_avx512vbmi_compress_block_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_avx512vbmi_compress_block_f32(u8 bit_width, u32 block_size); -Kernel select_avx512vbmi_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_avx512vbmi_compress_blocks_f32(u8 bit_width, u32 block_size); -Kernel select_avx512vbmi_compress_block_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_avx512vbmi_compress_block_f64(u8 bit_width, u32 block_size); -Kernel select_avx512vbmi_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_avx512vbmi_compress_blocks_f64(u8 bit_width, u32 block_size); -Kernel select_avx512vbmi_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_avx512vbmi_decompress_block_f32(u8 bit_width, u32 block_size, + bool sign_values); -Kernel select_avx512vbmi_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_avx512vbmi_decompress_blocks_f32(u8 bit_width, u32 block_size, + bool sign_values); -Kernel select_avx512vbmi_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_avx512vbmi_decompress_block_f64(u8 bit_width, u32 block_size, + bool sign_values); -Kernel select_avx512vbmi_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_avx512vbmi_decompress_blocks_f64(u8 bit_width, u32 block_size, + bool sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) -Kernel select_neon_compress_block_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_neon_compress_block_f32(u8 bit_width, u32 block_size); -Kernel select_neon_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_neon_compress_blocks_f32(u8 bit_width, u32 block_size); -Kernel select_neon_compress_block_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_neon_compress_block_f64(u8 bit_width, u32 block_size); -Kernel select_neon_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_neon_compress_blocks_f64(u8 bit_width, u32 block_size); -Kernel select_neon_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_neon_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_neon_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_neon_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_neon_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_neon_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_neon_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_neon_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) -Kernel select_sve2_compress_block_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_sve2_compress_block_f32(u8 bit_width, u32 block_size); -Kernel select_sve2_compress_blocks_f32(uint8_t bit_width, uint32_t block_size); + Kernel select_sve2_compress_blocks_f32(u8 bit_width, u32 block_size); -Kernel select_sve2_compress_block_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_sve2_compress_block_f64(u8 bit_width, u32 block_size); -Kernel select_sve2_compress_blocks_f64(uint8_t bit_width, uint32_t block_size); + Kernel select_sve2_compress_blocks_f64(u8 bit_width, u32 block_size); -Kernel select_sve2_decompress_block_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_sve2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_sve2_decompress_blocks_f32(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_sve2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_sve2_decompress_block_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_sve2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); -Kernel select_sve2_decompress_blocks_f64(uint8_t bit_width, uint32_t block_size, bool sign_values); + Kernel select_sve2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #endif } diff --git a/src/internal/pernix/fallback/avx2_compression.h b/src/internal/pernix/fallback/avx2_compression.h index c7f03bc..e2c7200 100644 --- a/src/internal/pernix/fallback/avx2_compression.h +++ b/src/internal/pernix/fallback/avx2_compression.h @@ -11,69 +11,69 @@ #include namespace pernix { -namespace internal { -/** - * @brief Quantize a single float value to int32_t using the provided scale. + namespace internal { + /** + * @brief Quantize a single float value to i32 using the provided scale. * * @param input input float value to be quantized. * @param scale scaling factor used during quantization. - * @return int32_t quantized integer value. + * @return i32 quantized integer value. */ -__always_inline int32_t quantize_ps_epi32(const float input, const float scale) { - return static_cast(std::lroundf(input * scale)); -} +__always_inline i32 quantize_ps_epi32(const float input, const float scale) { + return static_cast(std::lroundf(input * scale)); + } -/** - * @brief Quantize a single double value to int64_t using the provided scale. + /** + * @brief Quantize a single double value to i64 using the provided scale. * * @param input input double value to be quantized. * @param scale scaling factor used during quantization. - * @return int64_t quantized integer value. + * @return i64 quantized integer value. */ -__always_inline int64_t quantize_pd_epi64(const double_t input, const double_t scale) { - return std::llround(input * scale); -} +__always_inline i64 quantize_pd_epi64(const f64 input, const f64 scale) { + return std::llround(input * scale); + } -/** + /** * @brief Quantize and clamp without narrowing through an out-of-range integer type. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -__always_inline int32_t quantize_clamped(const T input, const T scale) { - constexpr int64_t min_value = BIT_WIDTH == 1 ? 0 : -(int64_t{1} << (BIT_WIDTH - 1)); - constexpr int64_t max_value = BIT_WIDTH == 1 ? 1 : ((int64_t{1} << (BIT_WIDTH - 1)) - 1); - - const long double scaled = static_cast(input) * static_cast(scale); - if (std::isnan(scaled)) { - return 0; - } - if (scaled <= static_cast(min_value)) { - return static_cast(min_value); - } - if (scaled >= static_cast(max_value)) { - return static_cast(max_value); - } - - return static_cast(std::llround(scaled)); -} + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +__always_inline i32 quantize_clamped(const T input, const T scale) { + constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); + constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); + + const long double scaled = static_cast(input) * static_cast(scale); + if (std::isnan(scaled)) { + return 0; + } + if (scaled <= static_cast(min_value)) { + return static_cast(min_value); + } + if (scaled >= static_cast(max_value)) { + return static_cast(max_value); + } + + return static_cast(std::llround(scaled)); + } -/** + /** * @brief Clamp a signed quantized value to the representable range of BIT_WIDTH bits. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline int32_t clamp_signed_quantized(const int64_t value) { - if constexpr (BIT_WIDTH == 1) { - // 1-bit fallback is treated as binary quantization (0/1). - return static_cast(std::clamp(value, 0, 1)); - } - - constexpr int32_t min_value = -(1 << (BIT_WIDTH - 1)); - constexpr int32_t max_value = (1 << (BIT_WIDTH - 1)) - 1; - return static_cast(std::clamp(value, min_value, max_value)); -} + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__always_inline i32 clamp_signed_quantized(const i64 value) { + if constexpr (BIT_WIDTH == 1) { + // 1-bit fallback is treated as binary quantization (0/1). + return static_cast(std::clamp(value, 0, 1)); + } + + constexpr i32 min_value = -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = (1 << (BIT_WIDTH - 1)) - 1; + return static_cast(std::clamp(value, min_value, max_value)); + } -/** + /** * @brief Append packed scalar values into an output buffer using the selected * storage width. * @@ -83,58 +83,61 @@ __always_inline int32_t clamp_signed_quantized(const int64_t value) { * @param bit_offset starting bit offset in the destination buffer. * @param destination pointer to the output buffer. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) -void pack_epi32_fallback_inner(const std::vector& input, const uint8_t bit_offset, uint8_t* __restrict__ destination) { - constexpr uint32_t bits_in_type = sizeof(T) * 8; - constexpr uint32_t bitmask = BIT_WIDTH == bits_in_type ? std::numeric_limits::max() : (1U << BIT_WIDTH) - 1U; - - std::size_t idx = 0; - std::size_t bits_in_buffer = bit_offset; - uint64_t buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) + void pack_epi32_fallback_inner(const std::vector &input, const u8 bit_offset, + u8 * __restrict__ destination) { + constexpr u32 bits_in_type = sizeof(T) * 8; + constexpr u32 bitmask = BIT_WIDTH == bits_in_type + ? std::numeric_limits::max() + : (1U << BIT_WIDTH) - 1U; + + std::size_t idx = 0; + std::size_t bits_in_buffer = bit_offset; + u64 buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; #pragma GCC unroll 64 - for (uint32_t raw_value : input) { - const uint32_t next_value = raw_value & bitmask; - - buffer |= static_cast(next_value) << bits_in_buffer; - bits_in_buffer += BIT_WIDTH; - - while (bits_in_buffer >= 8) { - destination[idx++] = static_cast(buffer & 0xFFU); - buffer >>= 8; - bits_in_buffer -= 8; + for (u32 raw_value: input) { + const u32 next_value = raw_value & bitmask; + + buffer |= static_cast(next_value) << bits_in_buffer; + bits_in_buffer += BIT_WIDTH; + + while (bits_in_buffer >= 8) { + destination[idx++] = static_cast(buffer & 0xFFU); + buffer >>= 8; + bits_in_buffer -= 8; + } + } + + if (bits_in_buffer > 0) { + destination[idx] = static_cast(buffer & 0xFFU); + } } - } - - if (bits_in_buffer > 0) { - destination[idx] = static_cast(buffer & 0xFFU); - } -} -/** - * @brief Pack a vector of uint32_t values into a compact byte representation using fallback scalar implementation. + /** + * @brief Pack a vector of u32 values into a compact byte representation using fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). * - * @param input vector of uint32_t values to be packed. + * @param input vector of u32 values to be packed. * @param destination pointer to the output buffer where packed bytes will be stored. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -void pack_epi32_fallback(const std::vector& input, uint8_t* __restrict__ destination) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } else { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } -} -} // namespace internal + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + void pack_epi32_fallback(const std::vector &input, u8 * __restrict__ destination) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::pack_epi32_fallback_inner(input, 0, destination); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::pack_epi32_fallback_inner(input, 0, destination); + } else { + return internal::pack_epi32_fallback_inner(input, 0, destination); + } + } + } // namespace internal -/** + /** * @brief Compress a single 512-bit block using fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -145,28 +148,29 @@ void pack_epi32_fallback(const std::vector& input, uint8_t* __restrict * @param output pointer to the output buffer where compressed bytes will be stored. * @return int status code (0 for success). */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_block_fallback(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + int compress_block_fallback(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - std::vector block_values(elements_per_block); + std::vector block_values(elements_per_block); #pragma GCC unroll 64 - for (uint32_t i = 0; i < elements_per_block; i++) { - const int32_t quantized = internal::quantize_clamped(input[i], scale); - block_values[i] = static_cast(quantized); - } + for (u32 i = 0; i < elements_per_block; i++) { + const i32 quantized = internal::quantize_clamped(input[i], scale); + block_values[i] = static_cast(quantized); + } - internal::pack_epi32_fallback(block_values, output); - return 0; -} + internal::pack_epi32_fallback(block_values, output); + return 0; + } -/** + /** * @brief Compress a single block of double values using the fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -176,28 +180,29 @@ int compress_block_fallback(const void* __restrict__ input_ptr, const float_t sc * @param output pointer to the output buffer where compressed bytes will be stored. * @return int status code (0 for success). */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_block_fallback(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + int compress_block_fallback(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - std::vector block_values(elements_per_block); + std::vector block_values(elements_per_block); #pragma GCC unroll 32 - for (uint32_t i = 0; i < elements_per_block; i++) { - const int32_t quantized = internal::quantize_clamped(input[i], scale); - block_values[i] = static_cast(quantized); - } + for (u32 i = 0; i < elements_per_block; i++) { + const i32 quantized = internal::quantize_clamped(input[i], scale); + block_values[i] = static_cast(quantized); + } - internal::pack_epi32_fallback(block_values, output); - return 0; -} + internal::pack_epi32_fallback(block_values, output); + return 0; + } -/** + /** * @brief Compress multiple 512-bit blocks using fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -209,25 +214,26 @@ int compress_block_fallback(const void* __restrict__ input_ptr, const double_t s * @param blocks number of 512-bit blocks to compress. * @return int status code (0 for success). */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_blocks_fallback(const void* __restrict__ input_ptr, float scale, void* __restrict__ output_ptr, uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const float_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - compress_block_fallback(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + int compress_blocks_fallback(const void * __restrict__ input_ptr, float scale, void * __restrict__ output_ptr, + u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const f32 *block_input = input; + u8 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + compress_block_fallback(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } - return 0; -} + return 0; + } -/** + /** * @brief Compress multiple blocks of double values using the fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -238,22 +244,23 @@ int compress_blocks_fallback(const void* __restrict__ input_ptr, float scale, vo * @param blocks number of blocks to compress. * @return int status code (0 for success). */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_blocks_fallback(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, - const unsigned int blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const double_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - compress_block_fallback(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + int compress_blocks_fallback(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr, + const unsigned int blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const f64 *block_input = input; + u8 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + compress_block_fallback(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } + return 0; } - return 0; -} } // namespace pernix #endif // PERNIX_FALLBACK_COMPRESSION_H diff --git a/src/internal/pernix/fallback/avx2_decompression.h b/src/internal/pernix/fallback/avx2_decompression.h index f8f4680..24431e5 100644 --- a/src/internal/pernix/fallback/avx2_decompression.h +++ b/src/internal/pernix/fallback/avx2_decompression.h @@ -9,50 +9,50 @@ #include namespace pernix { -namespace internal { -/** -* @brief Dequantize a single int32_t value to float using the provided scale. + namespace internal { + /** +* @brief Dequantize a single i32 value to float using the provided scale. * -* @param input input int32_t value to be dequantized. +* @param input input i32 value to be dequantized. * @param scale scaling factor used during quantization. * @return float dequantized float value. */ -__always_inline float dequantize_epi32(const int32_t input, const float scale) { - return static_cast(input) * scale; -} +__always_inline float dequantize_epi32(const i32 input, const float scale) { + return static_cast(input) * scale; + } -/** -* @brief Dequantize a single int64_t value to double using the provided scale. + /** +* @brief Dequantize a single i64 value to double using the provided scale. * -* @param input input int64_t value to be dequantized. +* @param input input i64 value to be dequantized. * @param scale scaling factor used during quantization. -* @return double_t dequantized double value. +* @return f64 dequantized double value. */ -__always_inline double_t dequantize_epi64(const int64_t input, const double_t scale) { - return static_cast(input) * scale; -} +__always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { + return static_cast(input) * scale; + } -/** + /** * @brief Sign-extend a packed integer value stored in the low bits of a 32-bit word. * * @tparam BIT_WIDTH number of significant bits in the encoded value. * @param value unsigned packed value. -* @return int32_t sign-extended value. +* @return i32 sign-extended value. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline auto sign_extend(const uint32_t value) -> int32_t { - if constexpr (BIT_WIDTH == 1) { - return static_cast(value & 1U); - } - - constexpr uint32_t sign_bit = uint32_t{1} << (BIT_WIDTH - 1); - constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1; - const uint32_t masked = value & mask; - return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); -} + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + __always_inline auto sign_extend(const u32 value) -> i32 { + if constexpr (BIT_WIDTH == 1) { + return static_cast(value & 1U); + } + + constexpr u32 sign_bit = u32{1} << (BIT_WIDTH - 1); + constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1; + const u32 masked = value & mask; + return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); + } -/** + /** * @brief Unpack bit-packed values from a typed input span into signed 32-bit integers. * * @tparam T unsigned integer type used to read the source buffer. @@ -61,70 +61,70 @@ __always_inline auto sign_extend(const uint32_t value) -> int32_t { * @param input pointer to the typed packed input buffer. * @param bit_offset starting bit offset in the first input word. * @param elements number of values to unpack. -* @return std::vector unpacked values. +* @return std::vector unpacked values. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) -__always_inline auto unpack_epi32_fallback_inner(const uint8_t* __restrict__ input, const uint8_t bit_offset, - const std::size_t elements) - -> std::vector { - constexpr uint32_t bits_in_type = sizeof(T) * 8; - constexpr uint32_t bitmask = BIT_WIDTH == bits_in_type - ? std::numeric_limits::max() - : (1U << BIT_WIDTH) - 1U; - - std::vector output(elements); - - std::size_t idx = 0; - uint8_t bits_in_buffer = 8 - bit_offset; - uint64_t buffer = static_cast(input[idx++]) >> bit_offset; + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) + __always_inline auto unpack_epi32_fallback_inner(const u8 * __restrict__ input, const u8 bit_offset, + const std::size_t elements) + -> std::vector { + constexpr u32 bits_in_type = sizeof(T) * 8; + constexpr u32 bitmask = BIT_WIDTH == bits_in_type + ? std::numeric_limits::max() + : (1U << BIT_WIDTH) - 1U; + + std::vector output(elements); + + std::size_t idx = 0; + u8 bits_in_buffer = 8 - bit_offset; + u64 buffer = static_cast(input[idx++]) >> bit_offset; #pragma GCC unroll 64 - for (uint32_t i = 0; i < elements; i++) { - while (BIT_WIDTH > bits_in_buffer) { - const auto next_value = static_cast(input[idx++]) << bits_in_buffer; - buffer |= next_value; - bits_in_buffer += 8; - } - - const uint32_t raw_value = static_cast(buffer & bitmask); - if constexpr (SIGN_VALUES) { - output[i] = sign_extend(raw_value); - } else { - output[i] = static_cast(raw_value); + for (u32 i = 0; i < elements; i++) { + while (BIT_WIDTH > bits_in_buffer) { + const auto next_value = static_cast(input[idx++]) << bits_in_buffer; + buffer |= next_value; + bits_in_buffer += 8; + } + + const u32 raw_value = static_cast(buffer & bitmask); + if constexpr (SIGN_VALUES) { + output[i] = sign_extend(raw_value); + } else { + output[i] = static_cast(raw_value); + } + + buffer >>= BIT_WIDTH; + bits_in_buffer -= BIT_WIDTH; + } + + return output; } - buffer >>= BIT_WIDTH; - bits_in_buffer -= BIT_WIDTH; - } - - return output; -} - -/** -* @brief Unpack packed int32_t values from the input buffer using fallback scalar implementation. + /** +* @brief Unpack packed i32 values from the input buffer using fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). * @tparam SIGN_VALUES whether the values are signed or unsigned. * @param input pointer to the start of the packed data. * @param elements number of elements to unpack. -* @return std::vector unpacked int32_t values. +* @return std::vector unpacked i32 values. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline auto unpack_epi32_fallback(const uint8_t* __restrict__ input, - const std::size_t elements) -> std::vector { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else { - return unpack_epi32_fallback_inner(input, 0, elements); - } -} -} // namespace internal + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + __always_inline auto unpack_epi32_fallback(const u8 * __restrict__ input, + const std::size_t elements) -> std::vector { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return unpack_epi32_fallback_inner(input, 0, elements); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return unpack_epi32_fallback_inner(input, 0, elements); + } else { + return unpack_epi32_fallback_inner(input, 0, elements); + } + } + } // namespace internal -/** + /** * @brief Decompress a single 512\-bit block using fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -134,27 +134,27 @@ __always_inline auto unpack_epi32_fallback(const uint8_t* __restrict__ input, * @param output pointer to the output buffer where decompressed float values will be stored. * @return int status code (0 for success). */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_block_fallback(const void* __restrict__ input_ptr, const float_t scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) + int decompress_block_fallback(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const std::vector block_values = internal::unpack_epi32_fallback( - input, elements_per_block); + const std::vector block_values = internal::unpack_epi32_fallback( + input, elements_per_block); #pragma GCC unroll 512 - for (uint32_t i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi32(block_values[i], scale); - } + for (u32 i = 0; i < elements_per_block; i++) { + output[i] = internal::dequantize_epi32(block_values[i], scale); + } - return 0; -} + return 0; + } -/** + /** * @brief Decompress a single block to double values using the fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -164,27 +164,27 @@ int decompress_block_fallback(const void* __restrict__ input_ptr, const float_t * @param output pointer to the output buffer where decompressed double values will be stored. * @return int status code (0 for success). */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_block_fallback(const void* __restrict__ input_ptr, const double_t scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) + int decompress_block_fallback(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const std::vector block_values = internal::unpack_epi32_fallback( - input, elements_per_block); + const std::vector block_values = internal::unpack_epi32_fallback( + input, elements_per_block); #pragma GCC unroll 512 - for (uint32_t i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi64(block_values[i], scale); - } + for (u32 i = 0; i < elements_per_block; i++) { + output[i] = internal::dequantize_epi64(block_values[i], scale); + } - return 0; -} + return 0; + } -/** + /** * @brief Decompress multiple 512\-bit blocks using fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -195,26 +195,26 @@ int decompress_block_fallback(const void* __restrict__ input_ptr, const double_t * @param blocks number of 512-bit blocks to decompress. * @return int status code (0 for success). */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int decompress_blocks_fallback(const void* __restrict__ input_ptr, const float_t scale, - void* __restrict__ output_ptr, const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + int decompress_blocks_fallback(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr, const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const u8 *block_input = input; + f32 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + decompress_block_fallback(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } - return 0; -} + return 0; + } -/** + /** * @brief Decompress multiple blocks to double values using the fallback scalar implementation. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -225,24 +225,24 @@ int decompress_blocks_fallback(const void* __restrict__ input_ptr, const float_t * @param blocks number of blocks to decompress. * @return int status code (0 for success). */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_blocks_fallback(const void* __restrict__ input_ptr, const double_t scale, - void* __restrict__ output_ptr, const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const uint8_t* block_input = input; - double_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) + int decompress_blocks_fallback(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr, const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const u8 *block_input = input; + f64 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + decompress_block_fallback(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } - return 0; -} + return 0; + } } // namespace pernix #endif // PERNIX_FALLBACK_DECOMPRESSION_H diff --git a/src/internal/pernix/simd_compat.h b/src/internal/pernix/simd_compat.h index b9dac8e..d55aafa 100644 --- a/src/internal/pernix/simd_compat.h +++ b/src/internal/pernix/simd_compat.h @@ -22,19 +22,6 @@ #include #endif -// #ifndef __mmask8 -// typedef uint8_t __mmask8; -// #endif -// #ifndef __mmask16 -// typedef uint16_t __mmask16; -// #endif -// #ifndef __mmask32 -// typedef uint32_t __mmask32; -// #endif -// #ifndef __mmask64 -// typedef uint64_t __mmask64; -// #endif - #elif defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) #include #elif defined(__aarch64__) || defined(__arm64ec__) diff --git a/src/internal/pernix/x86/avx2/avx2_compression.h b/src/internal/pernix/x86/avx2/avx2_compression.h index 3486f02..01c9626 100644 --- a/src/internal/pernix/x86/avx2/avx2_compression.h +++ b/src/internal/pernix/x86/avx2/avx2_compression.h @@ -11,68 +11,68 @@ #include namespace pernix { -namespace internal { -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + namespace internal { + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __always_inline __m256i mm256_clamp_signed_epi32(__m256i input) { - constexpr int32_t min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); - constexpr int32_t max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), - _mm256_set1_epi32(max_value)); -} + constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); + return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), + _mm256_set1_epi32(max_value)); + } -/** + /** * @brief Quantize four float values into signed 32-bit integers. * * @param input source float lane values. * @param scale per-lane scale factor. * @return __m128i quantized values. */ -__always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& scale) { - const __m128 scaled = _mm_mul_ps(input, scale); - // const __m128 rounded = _mm_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); - return _mm_cvtps_epi32(scaled); -} +__always_inline __m128i mm_quantize_ps_epi32(const __m128 &input, const __m128 &scale) { + const __m128 scaled = _mm_mul_ps(input, scale); + // const __m128 rounded = _mm_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + return _mm_cvtps_epi32(scaled); + } -/** + /** * @brief Quantize two double values into a partially filled 128-bit integer register. * * @param input source double lane values. * @param scale per-lane scale factor. * @return __m128i quantized values in the low lanes. */ -__always_inline __m128i mm_quantize_pd_epi32(const __m128d& input, const __m128d& scale) { - const __m128d scaled = _mm_mul_pd(input, scale); - return _mm_cvtpd_epi32(scaled); -} +__always_inline __m128i mm_quantize_pd_epi32(const __m128d &input, const __m128d &scale) { + const __m128d scaled = _mm_mul_pd(input, scale); + return _mm_cvtpd_epi32(scaled); + } -/** + /** * @brief Quantize eight float values into signed 32-bit integers. * * @param input source float lane values. * @param scale per-lane scale factor. * @return __m256i quantized values. */ -__always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m256& scale) { - const __m256 scaled = _mm256_mul_ps(input, scale); - // const __m256 rounded = _mm256_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); - return _mm256_cvtps_epi32(scaled); -} +__always_inline __m256i mm256_quantize_ps_epi32(const __m256 &input, const __m256 &scale) { + const __m256 scaled = _mm256_mul_ps(input, scale); + // const __m256 rounded = _mm256_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + return _mm256_cvtps_epi32(scaled); + } -/** + /** * @brief Quantize four double values into signed 32-bit integers. * * @param input source double lane values. * @param scale per-lane scale factor. * @return __m128i quantized values in the low lanes. */ -__always_inline __m128i mm256_quantize_pd_epi32(const __m256d& input, const __m256d& scale) { - const __m256d scaled = _mm256_mul_pd(input, scale); - return _mm256_cvtpd_epi32(scaled); -} +__always_inline __m128i mm256_quantize_pd_epi32(const __m256d &input, const __m256d &scale) { + const __m256d scaled = _mm256_mul_pd(input, scale); + return _mm256_cvtpd_epi32(scaled); + } #ifndef PERNIX_USE_SIMDE -/** + /** * @brief Emulate per-16-bit left shifts on AVX2. * * @param a source values. @@ -80,350 +80,350 @@ __always_inline __m128i mm256_quantize_pd_epi32(const __m256d& input, const __m2 * @return __m128i shifted values. */ __always_inline static __m128i _mm_sllv_epi16(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi32(0xffff0000); - const __m128i low_half = _mm_sllv_epi32(a, _mm_andnot_si128(mask, count)); - const __m128i high_half = _mm_sllv_epi32(_mm_and_si128(mask, a), _mm_srli_epi32(count, 16)); - return _mm_blend_epi16(low_half, high_half, 0xaa); -} + const __m128i mask = _mm_set1_epi32(0xffff0000); + const __m128i low_half = _mm_sllv_epi32(a, _mm_andnot_si128(mask, count)); + const __m128i high_half = _mm_sllv_epi32(_mm_and_si128(mask, a), _mm_srli_epi32(count, 16)); + return _mm_blend_epi16(low_half, high_half, 0xaa); + } -/** + /** * @brief Emulate per-16-bit right shifts on AVX2. */ __always_inline static __m128i _mm_srlv_epi16(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi32(0x0000ffff); - const __m128i low_half = _mm_srlv_epi32(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); - const __m128i high_half = _mm_srlv_epi32(a, _mm_srli_epi32(count, 16)); - return _mm_blend_epi16(low_half, high_half, 0xaa); -} + const __m128i mask = _mm_set1_epi32(0x0000ffff); + const __m128i low_half = _mm_srlv_epi32(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); + const __m128i high_half = _mm_srlv_epi32(a, _mm_srli_epi32(count, 16)); + return _mm_blend_epi16(low_half, high_half, 0xaa); + } -/** + /** * @brief Emulate per-16-bit left shifts on 256-bit AVX2 vectors. */ __always_inline static __m256i _mm256_sllv_epi16(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi32(0xffff0000); - const __m256i low_half = _mm256_sllv_epi32(a, _mm256_andnot_si256(mask, count)); - const __m256i high_half = _mm256_sllv_epi32(_mm256_and_si256(mask, a), _mm256_srli_epi32(count, 16)); - return _mm256_blend_epi16(low_half, high_half, 0xaa); -} + const __m256i mask = _mm256_set1_epi32(0xffff0000); + const __m256i low_half = _mm256_sllv_epi32(a, _mm256_andnot_si256(mask, count)); + const __m256i high_half = _mm256_sllv_epi32(_mm256_and_si256(mask, a), _mm256_srli_epi32(count, 16)); + return _mm256_blend_epi16(low_half, high_half, 0xaa); + } -/** + /** * @brief Emulate per-16-bit right shifts on 256-bit AVX2 vectors. */ __always_inline static __m256i _mm256_srlv_epi16(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi32(0x0000ffff); - const __m256i low_half = _mm256_srlv_epi32(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); - const __m256i high_half = _mm256_srlv_epi32(a, _mm256_srli_epi32(count, 16)); - return _mm256_blend_epi16(low_half, high_half, 0xaa); -} + const __m256i mask = _mm256_set1_epi32(0x0000ffff); + const __m256i low_half = _mm256_srlv_epi32(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); + const __m256i high_half = _mm256_srlv_epi32(a, _mm256_srli_epi32(count, 16)); + return _mm256_blend_epi16(low_half, high_half, 0xaa); + } -/** + /** * @brief Blend 8-bit lanes by expanding a scalar mask value. */ -__always_inline static __m128i mm_blend_epi8(const __m128i X, const __m128i Y, const int8_t M) { - return _mm_blendv_epi8(X, Y, _mm_set1_epi8(M)); -} +__always_inline static __m128i mm_blend_epi8(const __m128i X, const __m128i Y, const i8 M) { + return _mm_blendv_epi8(X, Y, _mm_set1_epi8(M)); + } -/** + /** * @brief Blend 8-bit lanes in 256-bit vectors by expanding a scalar mask value. */ -__always_inline static __m256i mm256_blend_epi8(const __m256i X, const __m256i Y, const int8_t M) { - return _mm256_blendv_epi8(X, Y, _mm256_set1_epi8(M)); -} +__always_inline static __m256i mm256_blend_epi8(const __m256i X, const __m256i Y, const i8 M) { + return _mm256_blendv_epi8(X, Y, _mm256_set1_epi8(M)); + } -/** + /** * @brief Emulate per-byte left shifts on 128-bit vectors. */ __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0xff00); - const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); - const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); - return mm_blend_epi8(low_half, high_half, 0xaa); -} + const __m128i mask = _mm_set1_epi16(0xff00); + const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); + const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); + return mm_blend_epi8(low_half, high_half, 0xaa); + } -/** + /** * @brief Emulate per-byte right shifts on 128-bit vectors. */ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0x00ff); - const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); - const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); - return mm_blend_epi8(low_half, high_half, 0xaa); -} + const __m128i mask = _mm_set1_epi16(0x00ff); + const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); + const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); + return mm_blend_epi8(low_half, high_half, 0xaa); + } -/** + /** * @brief Emulate per-byte left shifts on 256-bit vectors. */ __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0xff00); - const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); - const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); - return mm256_blend_epi8(low_half, high_half, 0xaa); -} + const __m256i mask = _mm256_set1_epi16(0xff00); + const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); + const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); + return mm256_blend_epi8(low_half, high_half, 0xaa); + } -/** + /** * @brief Emulate per-byte right shifts on 256-bit vectors. */ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0x00ff); - const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); - const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); - return mm256_blend_epi8(low_half, high_half, 0xaa); -} + const __m256i mask = _mm256_set1_epi16(0x00ff); + const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); + const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); + return mm256_blend_epi8(low_half, high_half, 0xaa); + } #endif -/** + /** * @brief Pack four 32-bit values for bit widths 1 through 3. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) -__always_inline auto mm_pack_epi32_avx2_1to3(__m128i& input) -> __m128i { - constexpr uint32_t bitmask = (1U << BIT_WIDTH) - 1U; - const __m128i masked = _mm_and_si128(input, _mm_set1_epi32(static_cast(bitmask))); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) + __always_inline auto mm_pack_epi32_avx2_1to3(__m128i &input) -> __m128i { + constexpr u32 bitmask = (1U << BIT_WIDTH) - 1U; + const __m128i masked = _mm_and_si128(input, _mm_set1_epi32(static_cast(bitmask))); - alignas(16) uint32_t lanes[4]; - _mm_storeu_si128(reinterpret_cast<__m128i*>(lanes), masked); + alignas(16) u32 lanes[4]; + _mm_storeu_si128(reinterpret_cast<__m128i *>(lanes), masked); - const uint32_t packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ( - (lanes[2] & bitmask) << (2 * BIT_WIDTH)) | - ((lanes[3] & bitmask) << (3 * BIT_WIDTH)); + const u32 packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ( + (lanes[2] & bitmask) << (2 * BIT_WIDTH)) | + ((lanes[3] & bitmask) << (3 * BIT_WIDTH)); - return _mm_cvtsi32_si128(static_cast(packed)); -} + return _mm_cvtsi32_si128(static_cast(packed)); + } -/** + /** * @brief Pack eight 32-bit values for bit widths 1 through 3. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) -__always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { - constexpr uint32_t bitmask = (1u << BIT_WIDTH) - 1u; + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) +__always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i &input) { + constexpr u32 bitmask = (1u << BIT_WIDTH) - 1u; - const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(static_cast(bitmask))); + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(static_cast(bitmask))); - const __m256i shifts = _mm256_setr_epi32(0 * BIT_WIDTH, 1 * BIT_WIDTH, 2 * BIT_WIDTH, 3 * BIT_WIDTH, - 4 * BIT_WIDTH, 5 * BIT_WIDTH, - 6 * BIT_WIDTH, 7 * BIT_WIDTH); + const __m256i shifts = _mm256_setr_epi32(0 * BIT_WIDTH, 1 * BIT_WIDTH, 2 * BIT_WIDTH, 3 * BIT_WIDTH, + 4 * BIT_WIDTH, 5 * BIT_WIDTH, + 6 * BIT_WIDTH, 7 * BIT_WIDTH); - const __m256i shifted = _mm256_sllv_epi32(masked, shifts); + const __m256i shifted = _mm256_sllv_epi32(masked, shifts); - __m128i x = _mm_or_si128(_mm256_castsi256_si128(shifted), _mm256_extracti128_si256(shifted, 1)); + __m128i x = _mm_or_si128(_mm256_castsi256_si128(shifted), _mm256_extracti128_si256(shifted, 1)); - x = _mm_or_si128(x, _mm_srli_si128(x, 8)); - x = _mm_or_si128(x, _mm_srli_si128(x, 4)); + x = _mm_or_si128(x, _mm_srli_si128(x, 8)); + x = _mm_or_si128(x, _mm_srli_si128(x, 4)); - return _mm256_castsi128_si256(x); -} + return _mm256_castsi128_si256(x); + } -__always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i& input) { - const __m256i zero = _mm256_setzero_si256(); +__always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i &input) { + const __m256i zero = _mm256_setzero_si256(); - const __m256i packed16 = _mm256_packus_epi32(input, zero); - const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); - const __m256i packed8 = _mm256_packus_epi16(permuted, zero); + const __m256i packed16 = _mm256_packus_epi32(input, zero); + const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); + const __m256i packed8 = _mm256_packus_epi16(permuted, zero); - const __m256i combined = _mm256_or_si256(packed8, _mm256_srli_epi16(packed8, 4)); + const __m256i combined = _mm256_or_si256(packed8, _mm256_srli_epi16(packed8, 4)); - const __m256i shuffled = _mm256_shuffle_epi8(combined, _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 2, - 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i shuffled = _mm256_shuffle_epi8(combined, _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 2, + 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1)); - return shuffled; -} + return shuffled; + } -/** + /** * @brief Pack four 32-bit values for bit widths 9 through 16. */ -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { - using tables = pack_tables_avx2_16; - constexpr uint16_t bitmask = (1 << BIT_WIDTH) - 1; - const __m128i masked = _mm_and_si128(input, _mm_set1_epi16(bitmask)); - __m128i combined; - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); - const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); - - const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); - const __m128i shifted2 = _mm_srlv_epi16(shuffled2, tables::get_shift2()); - - combined = _mm_or_si128(shifted1, shifted2); - } else { - const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); - const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); - const __m128i shuffled3 = _mm_shuffle_epi8(masked, tables::get_permute3()); - - const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi16(shuffled2, tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi16(shuffled3, tables::get_shift3()); - - combined = _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); - } - return combined; -} + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) + __always_inline auto mm_pack_epi32_avx2_9to16(__m128i &input) -> __m128i { + using tables = pack_tables_avx2_16; + constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; + const __m128i masked = _mm_and_si128(input, _mm_set1_epi16(bitmask)); + __m128i combined; + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); + const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); + + const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); + const __m128i shifted2 = _mm_srlv_epi16(shuffled2, tables::get_shift2()); + + combined = _mm_or_si128(shifted1, shifted2); + } else { + const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); + const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); + const __m128i shuffled3 = _mm_shuffle_epi8(masked, tables::get_permute3()); + + const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi16(shuffled2, tables::get_shift2()); + const __m128i shifted3 = _mm_srlv_epi16(shuffled3, tables::get_shift3()); + + combined = _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); + } + return combined; + } -/** + /** * @brief Pack eight 32-bit values for bit widths 9 through 16. */ -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i& input) -> __m256i { - using tables = pack_tables_avx2_16; - constexpr uint16_t bitmask = (1 << BIT_WIDTH) - 1; - const __m128i packed = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); - const __m128i masked = _mm_and_si128(packed, _mm_set1_epi16(bitmask)); - __m128i combined; - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15 || BIT_WIDTH == 16) { - const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); - const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); - - const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); - const __m128i shifted2 = _mm_srlv_epi16(shuffled2, tables::get_shift2()); - - combined = _mm_or_si128(shifted1, shifted2); - } else { - const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); - const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); - const __m128i shuffled3 = _mm_shuffle_epi8(masked, tables::get_permute3()); - - const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi16(shuffled2, tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi16(shuffled3, tables::get_shift3()); - - combined = _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); - } - return _mm256_castsi128_si256(combined); -} + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) + __always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i &input) -> __m256i { + using tables = pack_tables_avx2_16; + constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; + const __m128i packed = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); + const __m128i masked = _mm_and_si128(packed, _mm_set1_epi16(bitmask)); + __m128i combined; + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15 || BIT_WIDTH == 16) { + const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); + const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); + + const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); + const __m128i shifted2 = _mm_srlv_epi16(shuffled2, tables::get_shift2()); + + combined = _mm_or_si128(shifted1, shifted2); + } else { + const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); + const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); + const __m128i shuffled3 = _mm_shuffle_epi8(masked, tables::get_permute3()); + + const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi16(shuffled2, tables::get_shift2()); + const __m128i shifted3 = _mm_srlv_epi16(shuffled3, tables::get_shift3()); + + combined = _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); + } + return _mm256_castsi128_si256(combined); + } -template - requires(BIT_WIDTH >= 5 && BIT_WIDTH <= 7) -__always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i { - const __m256i zero = _mm256_setzero_si256(); + template + requires(BIT_WIDTH >= 5 && BIT_WIDTH <= 7) + __always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i &input) -> __m256i { + const __m256i zero = _mm256_setzero_si256(); - const __m256i packed16 = _mm256_packus_epi32(input, zero); - const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); - const __m256i packed8 = _mm256_packus_epi16(permuted, zero); + const __m256i packed16 = _mm256_packus_epi32(input, zero); + const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); + const __m256i packed8 = _mm256_packus_epi16(permuted, zero); - const __m256i even = _mm256_and_si256(packed8, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(packed8, _mm256_set1_epi16(0xFF00)); + const __m256i even = _mm256_and_si256(packed8, _mm256_set1_epi16(0x00FF)); + const __m256i odd = _mm256_and_si256(packed8, _mm256_set1_epi16(0xFF00)); - const __m256i pair16 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); - const __m256i extended = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(pair16)); + const __m256i pair16 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); + const __m256i extended = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(pair16)); - return mm256_pack_epi32_avx2_9to16<2 * BIT_WIDTH>(extended); -} + return mm256_pack_epi32_avx2_9to16<2 * BIT_WIDTH>(extended); + } -/** + /** * @brief Pack eight 32-bit values for bit widths 17 through 24. */ -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m256i { - using tables = pack_tables_avx2_24; - constexpr uint32_t bitmask = (1 << BIT_WIDTH) - 1; - const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(bitmask)); - __m256i combined; - - if constexpr (BIT_WIDTH == 24) { - const __m256i shuffled1 = _mm256_permutevar8x32_epi32(masked, tables::get_permute1()); - const __m256i shuffled2 = _mm256_permutevar8x32_epi32(masked, tables::get_permute2()); - - const __m256i shifted1 = _mm256_sllv_epi32(shuffled1, tables::get_shift1()); - const __m256i shifted2 = _mm256_srlv_epi32(shuffled2, tables::get_shift2()); - - combined = _mm256_or_si256(shifted1, shifted2); - } else { - const __m256i shuffled1 = _mm256_permutevar8x32_epi32(masked, tables::get_permute1()); - const __m256i shuffled2 = _mm256_permutevar8x32_epi32(masked, tables::get_permute2()); - const __m256i shuffled3 = _mm256_permutevar8x32_epi32(masked, tables::get_permute3()); - - const __m256i shifted1 = _mm256_sllv_epi32(shuffled1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi32(shuffled2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi32(shuffled3, tables::get_shift3()); - - combined = _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); - } - - return combined; -} + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) + __always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i &input) -> __m256i { + using tables = pack_tables_avx2_24; + constexpr u32 bitmask = (1 << BIT_WIDTH) - 1; + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(bitmask)); + __m256i combined; + + if constexpr (BIT_WIDTH == 24) { + const __m256i shuffled1 = _mm256_permutevar8x32_epi32(masked, tables::get_permute1()); + const __m256i shuffled2 = _mm256_permutevar8x32_epi32(masked, tables::get_permute2()); + + const __m256i shifted1 = _mm256_sllv_epi32(shuffled1, tables::get_shift1()); + const __m256i shifted2 = _mm256_srlv_epi32(shuffled2, tables::get_shift2()); + + combined = _mm256_or_si256(shifted1, shifted2); + } else { + const __m256i shuffled1 = _mm256_permutevar8x32_epi32(masked, tables::get_permute1()); + const __m256i shuffled2 = _mm256_permutevar8x32_epi32(masked, tables::get_permute2()); + const __m256i shuffled3 = _mm256_permutevar8x32_epi32(masked, tables::get_permute3()); + + const __m256i shifted1 = _mm256_sllv_epi32(shuffled1, tables::get_shift1()); + const __m256i shifted2 = _mm256_sllv_epi32(shuffled2, tables::get_shift2()); + const __m256i shifted3 = _mm256_srlv_epi32(shuffled3, tables::get_shift3()); + + combined = _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); + } + + return combined; + } -/** + /** * @brief Pack aligned 8-bit or 16-bit values from four 32-bit lanes. */ -template - requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { - if constexpr (BIT_WIDTH == 8) { - return _mm_packus_epi16(_mm_packs_epi32(input, _mm_setzero_si128()), _mm_setzero_si128()); - } else { - return _mm_packs_epi32(input, _mm_setzero_si128()); - } -} + template + requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) + auto mm_pack_aligned_epi32_avx2(__m128i &input) -> __m128i { + if constexpr (BIT_WIDTH == 8) { + return _mm_packus_epi16(_mm_packs_epi32(input, _mm_setzero_si128()), _mm_setzero_si128()); + } else { + return _mm_packs_epi32(input, _mm_setzero_si128()); + } + } -/** + /** * @brief Dispatch to the appropriate 128-bit AVX2 packer for the selected bit width. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 16) -auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { - return internal::mm_pack_epi32_avx2_1to3(input); - } else if constexpr (BIT_WIDTH >= 4 && BIT_WIDTH <= 8) { - // TODO: implementation for 4-8 bits - return _mm_setzero_si128(); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::mm_pack_epi32_avx2_9to16(input); - } else { - return _mm_setzero_si128(); - } -} + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 16) + auto mm_pack_epi32_avx2(__m128i &input) -> __m128i { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { + return internal::mm_pack_epi32_avx2_1to3(input); + } else if constexpr (BIT_WIDTH >= 4 && BIT_WIDTH <= 8) { + // TODO: implementation for 4-8 bits + return _mm_setzero_si128(); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::mm_pack_epi32_avx2_9to16(input); + } else { + return _mm_setzero_si128(); + } + } -/** + /** * @brief Pack aligned 8-bit or 16-bit values from eight 32-bit lanes. */ -template - requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -__m256i mm256_pack_aligned_epi32_avx2(const __m256i& input) { - if constexpr (BIT_WIDTH == 8) { - const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), - _mm256_extracti128_si256(input, 1)); - const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); - return _mm256_castsi128_si256(packed8); - } else { - return _mm256_castsi128_si256( - _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1))); - } -} + template + requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) + __m256i mm256_pack_aligned_epi32_avx2(const __m256i &input) { + if constexpr (BIT_WIDTH == 8) { + const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), + _mm256_extracti128_si256(input, 1)); + const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); + return _mm256_castsi128_si256(packed8); + } else { + return _mm256_castsi128_si256( + _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1))); + } + } -/** + /** * @brief Dispatch to the appropriate 256-bit AVX2 packer for the selected bit width. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__m256i mm256_pack_epi32_avx2(const __m256i& input) { - if constexpr (BIT_WIDTH == 8 || BIT_WIDTH == 16) { - return internal::mm256_pack_aligned_epi32_avx2(input); - } else { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { - return internal::mm256_pack_epi32_avx2_1to3(input); - } else if constexpr (BIT_WIDTH == 4) { - return mm256_pack_epi32_avx2_4(input); - } else if constexpr (BIT_WIDTH >= 5 && BIT_WIDTH <= 7) { - return internal::mm256_pack_epi32_avx2_5to7(input); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 15) { - return internal::mm256_pack_epi32_avx2_9to16(input); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::mm256_pack_epi32_avx2_17to24(input); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + __m256i mm256_pack_epi32_avx2(const __m256i &input) { + if constexpr (BIT_WIDTH == 8 || BIT_WIDTH == 16) { + return internal::mm256_pack_aligned_epi32_avx2(input); + } else { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { + return internal::mm256_pack_epi32_avx2_1to3(input); + } else if constexpr (BIT_WIDTH == 4) { + return mm256_pack_epi32_avx2_4(input); + } else if constexpr (BIT_WIDTH >= 5 && BIT_WIDTH <= 7) { + return internal::mm256_pack_epi32_avx2_5to7(input); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 15) { + return internal::mm256_pack_epi32_avx2_9to16(input); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::mm256_pack_epi32_avx2_17to24(input); + } + } + return _mm256_setzero_si256(); } - } - return _mm256_setzero_si256(); -} -} // namespace internal + } // namespace internal -/** + /** * @brief Compress a single block of float using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -436,48 +436,48 @@ __m256i mm256_pack_epi32_avx2(const __m256i& input) { * * @note This function requires AVX2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const float_t scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_compress_block_avx2(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - const __m256 scale_v = _mm256_set1_ps(scale); + const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 8 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256 source = _mm256_loadu_ps(input); - const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); - const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); - const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); - std::memcpy(output, &packed, BIT_WIDTH); - - input += 8; - output += BIT_WIDTH; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256 source = _mm256_loadu_ps(input); + const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + std::memcpy(output, &packed, BIT_WIDTH); + + input += 8; + output += BIT_WIDTH; + } - if constexpr (remaining) { - std::vector block_values(remaining); + if constexpr (remaining) { + std::vector block_values(remaining); #pragma GCC unroll 8 - for (uint32_t i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized( - internal::quantize_ps_epi32(input[i], scale))); + for (u32 i = 0; i < remaining; i++) { + block_values[i] = + static_cast(internal::clamp_signed_quantized < BIT_WIDTH > ( + internal::quantize_ps_epi32(input[i], scale))); + } + + internal::pack_epi32_fallback < BIT_WIDTH > (block_values, output); } - internal::pack_epi32_fallback(block_values, output); + return 0; } - return 0; -} - -/** + /** * @brief Compress a single block of double using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -490,52 +490,52 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const float_t * * @note This function requires AVX2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const double_t scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_compress_block_avx2(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - const __m256d scale_v = _mm256_set1_pd(scale); + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 8 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256d source1 = _mm256_loadu_pd(input); - const __m256d source2 = _mm256_loadu_pd(input + 4); - const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); - const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); - __m256i combined = _mm256_castsi128_si256(quantized1); - combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_avx2( - internal::mm256_clamp_signed_epi32(combined)); - // _mm_storeu_si128(reinterpret_cast<__m128i*>(output), _mm256_castsi256_si128(packed)); - std::memcpy(output, &packed, BIT_WIDTH); - input += 8; - output += BIT_WIDTH; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256d source1 = _mm256_loadu_pd(input); + const __m256d source2 = _mm256_loadu_pd(input + 4); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed = internal::mm256_pack_epi32_avx2( + internal::mm256_clamp_signed_epi32(combined)); + // _mm_storeu_si128(reinterpret_cast<__m128i*>(output), _mm256_castsi256_si128(packed)); + std::memcpy(output, &packed, BIT_WIDTH); + input += 8; + output += BIT_WIDTH; + } - if constexpr (remaining) { - std::vector block_values(remaining); + if constexpr (remaining) { + std::vector block_values(remaining); #pragma GCC unroll 8 - for (uint32_t i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized( - internal::quantize_pd_epi64(input[i], scale))); + for (u32 i = 0; i < remaining; i++) { + block_values[i] = + static_cast(internal::clamp_signed_quantized < BIT_WIDTH > ( + internal::quantize_pd_epi64(input[i], scale))); + } + + internal::pack_epi32_fallback < BIT_WIDTH > (block_values, output); } - internal::pack_epi32_fallback(block_values, output); + return 0; } - return 0; -} - -/** + /** * @brief Compress multiple blocks using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -549,27 +549,27 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const double_t * * @note This function requires AVX2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const float_t scale, - void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const float_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_compress_block_avx2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_compress_blocks_avx2(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const f32 *block_input = input; + u8 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_avx2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } - return 0; -} + return 0; + } -/** + /** * @brief Compress multiple blocks using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -583,25 +583,25 @@ int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const float_t * * @note This function requires AVX2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const double_t scale, - void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const double_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_compress_block_avx2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_compress_blocks_avx2(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const f64 *block_input = input; + u8 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_avx2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } - return 0; -} + return 0; + } } // namespace pernix #endif // PERNIX_AVX2_COMPRESSION_H diff --git a/src/internal/pernix/x86/avx2/avx2_decompression.h b/src/internal/pernix/x86/avx2/avx2_decompression.h index 7c530d8..2f50018 100644 --- a/src/internal/pernix/x86/avx2/avx2_decompression.h +++ b/src/internal/pernix/x86/avx2/avx2_decompression.h @@ -11,173 +11,176 @@ #include namespace pernix { -namespace internal { -/** + namespace internal { + /** * @brief Convert an 8-lane mask to the lane representation used by AVX2 float masked stores. */ __always_inline __m256i mm256_convert_vmask_epi32(const __mmask8 mask8) { - return _mm256_setr_epi32((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, (mask8 & 0x8) ? -1 : 0, - (mask8 & 0x10) ? -1 : 0, (mask8 & 0x20) ? -1 : 0, (mask8 & 0x40) ? -1 : 0, (mask8 & 0x80) ? -1 : 0); -} + return _mm256_setr_epi32((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, + (mask8 & 0x8) ? -1 : 0, + (mask8 & 0x10) ? -1 : 0, (mask8 & 0x20) ? -1 : 0, (mask8 & 0x40) ? -1 : 0, + (mask8 & 0x80) ? -1 : 0); + } -/** + /** * @brief Convert a 4-lane mask to the lane representation used by AVX2 double masked stores. */ __always_inline __m256i mm256_convert_vmask_epi64(const __mmask8 mask8) { - return _mm256_setr_epi64x((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, (mask8 & 0x8) ? -1 : 0); -} + return _mm256_setr_epi64x((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, + (mask8 & 0x8) ? -1 : 0); + } -/** + /** * @brief Dequantize four 32-bit integers to floats. */ -__always_inline __m128 mm_dequantize_epi32(const __m128i& input, const __m128& scale) { - const __m128 converted = _mm_cvtepi32_ps(input); - return _mm_mul_ps(converted, scale); -} +__always_inline __m128 mm_dequantize_epi32(const __m128i &input, const __m128 &scale) { + const __m128 converted = _mm_cvtepi32_ps(input); + return _mm_mul_ps(converted, scale); + } -/* https://stackoverflow.com/questions/41144668/how-to-efficiently-perform-double-int64-conversions-with-sse-avx */ + /* https://stackoverflow.com/questions/41144668/how-to-efficiently-perform-double-int64-conversions-with-sse-avx */ __always_inline __m128d convert_epi64_pd(const __m128i v) { - __m128i xH = _mm_srai_epi32(v, 16); - xH = _mm_blend_epi16(xH, _mm_setzero_si128(), 0x33); - xH = _mm_add_epi64(xH, _mm_castpd_si128(_mm_set1_pd(442721857769029238784.))); // 3*2^67 - const __m128i xL = _mm_blend_epi16(v, _mm_castpd_si128(_mm_set1_pd(0x0010000000000000)), 0x88); // 2^52 - const __m128d f = _mm_sub_pd(_mm_castsi128_pd(xH), _mm_set1_pd(442726361368656609280.)); // 3*2^67 + 2^52 - return _mm_add_pd(f, _mm_castsi128_pd(xL)); -} - -/** + __m128i xH = _mm_srai_epi32(v, 16); + xH = _mm_blend_epi16(xH, _mm_setzero_si128(), 0x33); + xH = _mm_add_epi64(xH, _mm_castpd_si128(_mm_set1_pd(442721857769029238784.))); // 3*2^67 + const __m128i xL = _mm_blend_epi16(v, _mm_castpd_si128(_mm_set1_pd(0x0010000000000000)), 0x88); // 2^52 + const __m128d f = _mm_sub_pd(_mm_castsi128_pd(xH), _mm_set1_pd(442726361368656609280.)); // 3*2^67 + 2^52 + return _mm_add_pd(f, _mm_castsi128_pd(xL)); + } + + /** * @brief Dequantize two 64-bit integers to doubles. */ -__always_inline __m128d mm_dequantize_epi64_pd(const __m128i& input, const __m128d& scale) { - const __m128d converted = convert_epi64_pd(input); - return _mm_mul_pd(converted, scale); -} +__always_inline __m128d mm_dequantize_epi64_pd(const __m128i &input, const __m128d &scale) { + const __m128d converted = convert_epi64_pd(input); + return _mm_mul_pd(converted, scale); + } -/** + /** * @brief Dequantize eight 32-bit integers to floats. */ -__always_inline __m256 mm256_dequantize_epi32(const __m256i& input, const __m256& scale) { - const __m256 converted = _mm256_cvtepi32_ps(input); - return _mm256_mul_ps(converted, scale); -} +__always_inline __m256 mm256_dequantize_epi32(const __m256i &input, const __m256 &scale) { + const __m256 converted = _mm256_cvtepi32_ps(input); + return _mm256_mul_ps(converted, scale); + } -/* https://stackoverflow.com/questions/41144668/how-to-efficiently-perform-double-int64-conversions-with-sse-avx */ + /* https://stackoverflow.com/questions/41144668/how-to-efficiently-perform-double-int64-conversions-with-sse-avx */ __always_inline __m256d convert_epi64_pd(__m256i v) { - __m256i xH = _mm256_srai_epi32(v, 16); - xH = _mm256_blend_epi16(xH, _mm256_setzero_si256(), 0x33); - xH = _mm256_add_epi64(xH, _mm256_castpd_si256(_mm256_set1_pd(442721857769029238784.))); - const __m256i xL = _mm256_blend_epi16(v, _mm256_castpd_si256(_mm256_set1_pd(0x0010000000000000)), 0x88); - const __m256d f = _mm256_sub_pd(_mm256_castsi256_pd(xH), _mm256_set1_pd(442726361368656609280.)); - return _mm256_add_pd(f, _mm256_castsi256_pd(xL)); -} - -/** + __m256i xH = _mm256_srai_epi32(v, 16); + xH = _mm256_blend_epi16(xH, _mm256_setzero_si256(), 0x33); + xH = _mm256_add_epi64(xH, _mm256_castpd_si256(_mm256_set1_pd(442721857769029238784.))); + const __m256i xL = _mm256_blend_epi16(v, _mm256_castpd_si256(_mm256_set1_pd(0x0010000000000000)), 0x88); + const __m256d f = _mm256_sub_pd(_mm256_castsi256_pd(xH), _mm256_set1_pd(442726361368656609280.)); + return _mm256_add_pd(f, _mm256_castsi256_pd(xL)); + } + + /** * @brief Dequantize four 64-bit integers to doubles. */ -__always_inline __m256d mm256_dequantize_epi64_pd(const __m256i& input, const __m256d& scale) { - const __m256d converted = convert_epi64_pd(input); - return _mm256_mul_pd(converted, scale); -} +__always_inline __m256d mm256_dequantize_epi64_pd(const __m256i &input, const __m256d &scale) { + const __m256d converted = convert_epi64_pd(input); + return _mm256_mul_pd(converted, scale); + } -/** + /** * @brief Unpack four aligned 8-bit or 16-bit values directly from the input buffer. */ -template - requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -__m128i mm_unpack_aligned_epi32_avx2(const uint8_t* __restrict__ input) { - if constexpr (BIT_WIDTH == 8) { - const __m128i source = _mm_loadu_si32(input); - if constexpr (SIGN_VALUES) { - return _mm_cvtepi8_epi32(source); - } else { - return _mm_cvtepu8_epi32(source); - } - } else { - const __m128i source = _mm_loadu_si64(input); - if constexpr (SIGN_VALUES) { - return _mm_cvtepi16_epi32(source); - } else { - return _mm_cvtepu16_epi32(source); + template + requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) + __m128i mm_unpack_aligned_epi32_avx2(const u8 * __restrict__ input) { + if constexpr (BIT_WIDTH == 8) { + const __m128i source = _mm_loadu_si32(input); + if constexpr (SIGN_VALUES) { + return _mm_cvtepi8_epi32(source); + } else { + return _mm_cvtepu8_epi32(source); + } + } else { + const __m128i source = _mm_loadu_si64(input); + if constexpr (SIGN_VALUES) { + return _mm_cvtepi16_epi32(source); + } else { + return _mm_cvtepu16_epi32(source); + } + } } - } -} -/** + /** * @brief Unpack four values using the table-driven AVX2 shuffle path. */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -__m128i mm_unpack_epi32_avx2(const uint8_t* __restrict__ input) { - using unpack_table = unpack_tables_avx2; - constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; - - __m128i source = _mm_setzero_si128(); - std::memcpy(&source, input, packed_bytes); - - const __m128i shuffled = _mm_shuffle_epi8(source, unpack_table::get_shuffle()); - - constexpr uint16_t shift = 32 - BIT_WIDTH; - __m128i shifted = _mm_sllv_epi32(shuffled, unpack_table::get_shift()); - if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { - shifted = _mm_srai_epi32(shifted, shift); - } else { - shifted = _mm_srli_epi32(shifted, shift); - } - - return shifted; -} + template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) + __m128i mm_unpack_epi32_avx2(const u8 * __restrict__ input) { + using unpack_table = unpack_tables_avx2; + constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; + + __m128i source = _mm_setzero_si128(); + std::memcpy(&source, input, packed_bytes); + + const __m128i shuffled = _mm_shuffle_epi8(source, unpack_table::get_shuffle()); + + constexpr u16 shift = 32 - BIT_WIDTH; + __m128i shifted = _mm_sllv_epi32(shuffled, unpack_table::get_shift()); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + shifted = _mm_srai_epi32(shifted, shift); + } else { + shifted = _mm_srli_epi32(shifted, shift); + } + + return shifted; + } -/** + /** * @brief Unpack eight aligned 8-bit or 16-bit values directly from the input buffer. */ -template - requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -__m256i mm256_unpack_aligned_epi32_avx2(const uint8_t* __restrict__ input) { - if constexpr (BIT_WIDTH == 8) { - const __m128i source = _mm_loadu_si64(input); - if constexpr (SIGN_VALUES) { - return _mm256_cvtepi8_epi32(source); - } else { - return _mm256_cvtepu8_epi32(source); + template + requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) + __m256i mm256_unpack_aligned_epi32_avx2(const u8 * __restrict__ input) { + if constexpr (BIT_WIDTH == 8) { + const __m128i source = _mm_loadu_si64(input); + if constexpr (SIGN_VALUES) { + return _mm256_cvtepi8_epi32(source); + } else { + return _mm256_cvtepu8_epi32(source); + } + } else { + const __m128i source = _mm_loadu_si128(reinterpret_cast(input)); + if constexpr (SIGN_VALUES) { + return _mm256_cvtepi16_epi32(source); + } else { + return _mm256_cvtepu16_epi32(source); + } + } } - } else { - const __m128i source = _mm_loadu_si128(reinterpret_cast(input)); - if constexpr (SIGN_VALUES) { - return _mm256_cvtepi16_epi32(source); - } else { - return _mm256_cvtepu16_epi32(source); - } - } -} -/** + /** * @brief Unpack eight values using the table-driven AVX2 shuffle path. */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -__m256i mm256_unpack_epi32_avx2(const uint8_t* __restrict__ input) { - using unpack_table = unpack_tables_avx2; - constexpr std::size_t packed_bytes = BIT_WIDTH; - - __m256i source = _mm256_setzero_si256(); - std::memcpy(&source, input, packed_bytes); - - const __m256i permuted = _mm256_permutevar8x32_epi32(source, unpack_table::get_permute()); - const __m256i shuffled = _mm256_shuffle_epi8(permuted, unpack_table::get_shuffle()); - - constexpr uint16_t shift = 32 - BIT_WIDTH; - __m256i shifted = _mm256_sllv_epi32(shuffled, unpack_table::get_shift()); - if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { - shifted = _mm256_srai_epi32(shifted, shift); - } else { - shifted = _mm256_srli_epi32(shifted, shift); - } - - return shifted; -} -} // namespace internal + template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) + __m256i mm256_unpack_epi32_avx2(const u8 * __restrict__ input) { + using unpack_table = unpack_tables_avx2; + constexpr std::size_t packed_bytes = BIT_WIDTH; + + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, packed_bytes); + + const __m256i permuted = _mm256_permutevar8x32_epi32(source, unpack_table::get_permute()); + const __m256i shuffled = _mm256_shuffle_epi8(permuted, unpack_table::get_shuffle()); + + constexpr u16 shift = 32 - BIT_WIDTH; + __m256i shifted = _mm256_sllv_epi32(shuffled, unpack_table::get_shift()); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + shifted = _mm256_srai_epi32(shifted, shift); + } else { + shifted = _mm256_srli_epi32(shifted, shift); + } + + return shifted; + } + } // namespace internal -/** + /** * @brief Decompress a single block to float using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -190,37 +193,40 @@ __m256i mm256_unpack_epi32_avx2(const uint8_t* __restrict__ input) { * * @note This function requires AVX2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; - - const __m256 scale_v = _mm256_set1_ps(scale); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_decompress_block_avx2(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + + const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); - const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); - _mm256_storeu_ps(output, dequantized); - input += BIT_WIDTH; - output += 8; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); + const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); + _mm256_storeu_ps(output, dequantized); + input += BIT_WIDTH; + output += 8; + } - if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (uint32_t i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi32(tail_values[i], scale); + if constexpr (remaining > 0) { + const std::vector tail_values = internal::unpack_epi32_fallback < BIT_WIDTH, SIGN_VALUES + > + (input, remaining); + for (u32 i = 0; i < remaining; i++) { + output[i] = internal::dequantize_epi32(tail_values[i], scale); + } } - } - return 0; -} + return 0; + } -/** + /** * @brief Decompress a single block to double using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -233,42 +239,45 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const float_ * * @note This function requires AVX2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; - const __m256d scale_v = _mm256_set1_pd(scale); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_decompress_block_avx2(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); - const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); - const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); + const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); + const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); - const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); - const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); + const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); + const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); - _mm256_storeu_pd(output, dequantized1); - _mm256_storeu_pd(output + 4, dequantized2); + _mm256_storeu_pd(output, dequantized1); + _mm256_storeu_pd(output + 4, dequantized2); - input += BIT_WIDTH; - output += 8; - } + input += BIT_WIDTH; + output += 8; + } - if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (uint32_t i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi64(tail_values[i], scale); + if constexpr (remaining > 0) { + const std::vector tail_values = internal::unpack_epi32_fallback < BIT_WIDTH, SIGN_VALUES + > + (input, remaining); + for (u32 i = 0; i < remaining; i++) { + output[i] = internal::dequantize_epi64(tail_values[i], scale); + } } + return 0; } - return 0; -} -/** + /** * @brief Decompress multiple blocks to float using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -282,26 +291,27 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const double * * @note This function requires AVX2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_decompress_block_avx2(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_decompress_blocks_avx2(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const u8 *block_input = input; + f32 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_decompress_block_avx2(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } - return 0; -} + return 0; + } -/** + /** * @brief Decompress multiple blocks to double using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -315,24 +325,25 @@ int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const float * * @note This function requires AVX2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const uint8_t* block_input = input; - double_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_decompress_block_avx2(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_decompress_blocks_avx2(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const u8 *block_input = input; + f64 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_decompress_block_avx2(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } - return 0; -} + return 0; + } } // namespace pernix #endif // PERNIX_AVX2_DECOMPRESSION_H diff --git a/src/internal/pernix/x86/avx2/avx2_tables.h b/src/internal/pernix/x86/avx2/avx2_tables.h index e21af27..f62250f 100644 --- a/src/internal/pernix/x86/avx2/avx2_tables.h +++ b/src/internal/pernix/x86/avx2/avx2_tables.h @@ -7,13 +7,13 @@ #include namespace pernix::internal { - template<__uint8_t BIT_WIDTH, typename T> - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && (std::is_same_v || std::is_same_v)) - struct pack_tables_avx2_16 { - alignas(64) inline static constexpr std::array permute1 = [] { +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && (std::is_same_v || std::is_same_v)) +struct pack_tables_avx2_16 { + alignas(64) inline static constexpr std::array permute1 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 0, 1, 4, 5, 8, 9, 12, 13, -1, -1, -1, -1, @@ -25,7 +25,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 0, 1, 4, 5, -1, -1, 10, 11, 14, 15, -1, -1, @@ -37,7 +37,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 0, 1, 4, 5, 6, 7, 10, 11, 12, 13, -1, -1, @@ -49,7 +49,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 12) { - return std::array{ + return std::array{ 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, @@ -61,7 +61,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 0, 1, -1, -1, 6, 7, 8, 9, 10, 11, -1, -1, @@ -73,7 +73,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 14) { - return std::array{ + return std::array{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, @@ -85,7 +85,7 @@ namespace pernix::internal { 14, 15, -1, -1 }; } else if constexpr (BIT_WIDTH == 15) { - return std::array{ + return std::array{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, @@ -97,14 +97,14 @@ namespace pernix::internal { 14, 15, -1, -1 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute2 = [] { + alignas(64) inline static constexpr std::array permute2 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 2, 3, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, @@ -116,7 +116,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 2, 3, 6, 7, 8, 9, 12, 13, -1, -1, -1, -1, @@ -128,7 +128,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 2, 3, -1, -1, 8, 9, -1, -1, 14, 15, -1, -1, @@ -140,7 +140,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 12) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, @@ -152,7 +152,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 2, 3, 4, 5, -1, -1, -1, -1, 12, 13, 14, 15, @@ -164,7 +164,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 14) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, @@ -176,7 +176,7 @@ namespace pernix::internal { 12, 13, -1, -1 }; } else if constexpr (BIT_WIDTH == 15) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, @@ -188,14 +188,14 @@ namespace pernix::internal { 12, 13, 14, 15 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute3 = [] { + alignas(64) inline static constexpr std::array permute3 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ -1, -1, 2, 3, 6, 7, 10, 11, 14, 15, -1, -1, @@ -207,7 +207,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ -1, -1, 2, 3, 6, 7, 8, 9, 12, 13, -1, -1, @@ -219,7 +219,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ -1, -1, 2, 3, 4, 5, 8, 9, 10, 11, 14, 15, @@ -231,7 +231,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, @@ -243,183 +243,183 @@ namespace pernix::internal { 14, 15, -1, -1 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift1 = [] { + alignas(64) inline static constexpr std::array shift1 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 0, 2, 4, 6, 0, 1, 3, 5, 0, 2, 4, 6, 0, 1, 3, 5 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 0, 4, 0, 2, 6, 0, 4, 0, 0, 4, 0, 2, 6, 0, 4, 0 }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 0, 6, 1, 7, 2, 0, 3, 0, 0, 6, 1, 7, 2, 0, 3, 0 }; } else if constexpr (BIT_WIDTH == 12) { - return std::array{ + return std::array{ 12, 8, 4, 12, 8, 4, 12, 8, 12, 8, 4, 12, 8, 4, 12, 8 }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 0, 0, 7, 4, 1, 0, 0, 5, 0, 0, 7, 4, 1, 0, 0, 5 }; } else if constexpr (BIT_WIDTH == 14) { - return std::array{ + return std::array{ 14, 12, 10, 8, 6, 4, 2, 14, 14, 12, 10, 8, 6, 4, 2, 14 }; } else if constexpr (BIT_WIDTH == 15) { - return std::array{ + return std::array{ 15, 14, 13, 12, 11, 10, 9, 8, 15, 14, 13, 12, 11, 10, 9, 8 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift2 = [] { + alignas(64) inline static constexpr std::array shift2 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 9, 11, 13, 15, 8, 10, 12, 14, 9, 11, 13, 15, 8, 10, 12, 14, }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 10, 14, 8, 12, 0, 10, 14, 8, 10, 14, 8, 12, 0, 10, 14, 8, }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 11, 8, 12, 8, 13, 8, 14, 9, 11, 8, 12, 8, 13, 8, 14, 9, }; } else if constexpr (BIT_WIDTH == 12) { - return std::array{ + return std::array{ 0, 4, 8, 0, 4, 8, 0, 4, 0, 4, 8, 0, 4, 8, 0, 4, }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 13, 10, 0, 0, 14, 11, 8, 0, 13, 10, 0, 0, 14, 11, 8, 0, }; } else if constexpr (BIT_WIDTH == 14) { - return std::array{ + return std::array{ 0, 2, 4, 6, 8, 10, 12, 0, 0, 2, 4, 6, 8, 10, 12, 0, }; } else if constexpr (BIT_WIDTH == 15) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift3 = [] { + alignas(64) inline static constexpr std::array shift3 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 0, 7, 5, 3, 1, 8, 6, 4, 0, 7, 5, 3, 1, 8, 6, 4 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 0, 6, 2, 8, 4, 0, 6, 2, 0, 6, 2, 8, 4, 0, 6, 2, }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 0, 5, 10, 4, 9, 3, 8, 2, 0, 5, 10, 4, 9, 3, 8, 2, }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 0, 3, 6, 9, 12, 2, 5, 8, 0, 3, 6, 9, 12, 2, 5, 8, }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" __always_inline static T get_permute1() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute1.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute1.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute1.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute1.data())); } + return T{}; + } __always_inline static T get_permute2() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute2.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute2.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute2.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute2.data())); } + return T{}; + } __always_inline static T get_permute3() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute3.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute3.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute3.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute3.data())); } + return T{}; + } __always_inline static T get_shift1() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift1.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift1.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift1.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift1.data())); } + return T{}; + } __always_inline static T get_shift2() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift2.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift2.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift2.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift2.data())); } + return T{}; + } __always_inline static T get_shift3() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift3.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift3.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift3.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift3.data())); } + return T{}; + } #pragma GCC diagnostic pop - }; +}; - template<__uint8_t BIT_WIDTH, typename T> - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) - struct pack_tables_avx2_24 { - alignas(64) inline static constexpr std::array permute1 = [] { +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) +struct pack_tables_avx2_24 { + alignas(64) inline static constexpr std::array permute1 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -454,11 +454,11 @@ namespace pernix::internal { 1, 2, 3, 5, 6, 7, -1, -1, }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute2 = [] { + alignas(64) inline static constexpr std::array permute2 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -493,11 +493,11 @@ namespace pernix::internal { 0, 1, 2, 4, 5, 6, -1, -1, }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute3 = [] { + alignas(64) inline static constexpr std::array permute3 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -528,185 +528,185 @@ namespace pernix::internal { 0, 1, 2, 4, 5, 6, 0, 0 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift1 = [] { + alignas(64) inline static constexpr std::array shift1 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { - return std::array{ + return std::array{ 0, 2, 4, 6, 8, 10, 12, 14 }; } else if constexpr (BIT_WIDTH == 18) { - return std::array{ + return std::array{ 0, 4, 8, 12, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 19) { - return std::array{ + return std::array{ 0, 6, 12, 18, 5, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 20) { - return std::array{ + return std::array{ 0, 8, 16, 4, 12, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 21) { - return std::array{ + return std::array{ 0, 10, 20, 9, 19, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 22) { - return std::array{ + return std::array{ 0, 12, 2, 14, 4, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 23) { - return std::array{ + return std::array{ 0, 14, 5, 19, 10, 1, 32, 32 }; } else if constexpr (BIT_WIDTH == 24) { - return std::array{ + return std::array{ 24, 16, 8, 24, 16, 8, 24, 16 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift2 = [] { + alignas(64) inline static constexpr std::array shift2 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { - return std::array{ + return std::array{ 17, 19, 21, 23, 25, 27, 29, 31 }; } else if constexpr (BIT_WIDTH == 18) { - return std::array{ + return std::array{ 18, 22, 26, 30, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 19) { - return std::array{ + return std::array{ 19, 25, 31, 32, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 20) { - return std::array{ + return std::array{ 20, 28, 32, 24, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 21) { - return std::array{ + return std::array{ 21, 31, 32, 30, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 22) { - return std::array{ + return std::array{ 22, 32, 24, 32, 26, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 23) { - return std::array{ + return std::array{ 23, 32, 28, 32, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 24) { - return std::array{ + return std::array{ 0, 8, 16, 0, 8, 16, 0, 8, }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift3 = [] { + alignas(64) inline static constexpr std::array shift3 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { - return std::array{ + return std::array{ 0, 15, 13, 11, 9, 7, 5, 3 }; } else if constexpr (BIT_WIDTH == 18) { - return std::array{ + return std::array{ 32, 14, 10, 6, 2, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 19) { - return std::array{ + return std::array{ 32, 13, 7, 1, 14, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 20) { - return std::array{ + return std::array{ 32, 12, 4, 16, 8, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 21) { - return std::array{ + return std::array{ 32, 11, 1, 12, 2, 13, 32, 32 }; } else if constexpr (BIT_WIDTH == 22) { - return std::array{ + return std::array{ 32, 10, 20, 8, 18, 6, 32, 32 }; } else if constexpr (BIT_WIDTH == 23) { - return std::array{ + return std::array{ 32, 9, 18, 4, 13, 22, 32, 32 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" __always_inline static T get_permute1() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute1.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute1.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute1.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute1.data())); } + return T{}; + } __always_inline static T get_permute2() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute2.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute2.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute2.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute2.data())); } + return T{}; + } __always_inline static T get_permute3() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(permute3.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(permute3.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(permute3.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(permute3.data())); } + return T{}; + } __always_inline static T get_shift1() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift1.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift1.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift1.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift1.data())); } + return T{}; + } __always_inline static T get_shift2() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift2.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift2.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift2.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift2.data())); } + return T{}; + } __always_inline static T get_shift3() { - if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(shift3.data())); - } else if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift3.data())); - } - return T{}; + if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(shift3.data())); + } else if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift3.data())); } + return T{}; + } #pragma GCC diagnostic pop - }; +}; - template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) - struct unpack_tables_avx2 { - alignas(32) inline static constexpr std::array permute = [] { +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) +struct unpack_tables_avx2 { + alignas(32) inline static constexpr std::array permute = [] { // clang-format off if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return std::array{0, -1, -1, -1, 0, 1, -1, -1}; @@ -715,72 +715,72 @@ namespace pernix::internal { } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { return std::array{0, 1, 2, -1, 2, 3, 4, 5}; } - // clang-format on - }(); - - alignas(32) inline static constexpr std::array shuffle = [] { - std::array shuffles{}; - shuffles.fill(-1); - constexpr std::size_t rebase_second_half = 4 * ((BIT_WIDTH - 1) / 8); - - for (std::size_t lane = 0; lane < 2; ++lane) { - for (std::size_t i = 0; i < 4; ++i) { - const std::size_t value_index = lane * 4 + i; - - const std::size_t bit_start = value_index * BIT_WIDTH; - const std::size_t byte_start = bit_start / 8; - const std::size_t bit_offset = bit_start % 8; - const std::size_t byte_count = (bit_offset + BIT_WIDTH + 7) / 8; - - const std::size_t rebase = (lane == 0) ? 0 : rebase_second_half; - const std::size_t rel_byte_start = byte_start - rebase; - - const std::size_t dst = (lane * 4 + i) * 4; - for (std::size_t k = 0; k < byte_count; ++k) { - shuffles[dst + k] = static_cast(rel_byte_start + k); - } + // clang-format on + }(); + + alignas(32) inline static constexpr std::array shuffle = [] { + std::array shuffles{}; + shuffles.fill(-1); + constexpr std::size_t rebase_second_half = 4 * ((BIT_WIDTH - 1) / 8); + + for (std::size_t lane = 0; lane < 2; ++lane) { + for (std::size_t i = 0; i < 4; ++i) { + const std::size_t value_index = lane * 4 + i; + + const std::size_t bit_start = value_index * BIT_WIDTH; + const std::size_t byte_start = bit_start / 8; + const std::size_t bit_offset = bit_start % 8; + const std::size_t byte_count = (bit_offset + BIT_WIDTH + 7) / 8; + + const std::size_t rebase = (lane == 0) ? 0 : rebase_second_half; + const std::size_t rel_byte_start = byte_start - rebase; + + const std::size_t dst = (lane * 4 + i) * 4; + for (std::size_t k = 0; k < byte_count; ++k) { + shuffles[dst + k] = static_cast(rel_byte_start + k); } } + } - return shuffles; - }(); + return shuffles; + }(); - alignas(64) inline static constexpr std::array shift = [] { - std::array shifts{}; + alignas(64) inline static constexpr std::array shift = [] { + std::array shifts{}; - for (std::size_t lane = 0; lane < 8; ++lane) { - const int bit_offset = lane * BIT_WIDTH; - const int bit_in_byte = bit_offset % 8; - const int left_shift = 32 - BIT_WIDTH - bit_in_byte; - shifts[lane] = left_shift; - } + for (std::size_t lane = 0; lane < 8; ++lane) { + const int bit_offset = lane * BIT_WIDTH; + const int bit_in_byte = bit_offset % 8; + const int left_shift = 32 - BIT_WIDTH - bit_in_byte; + shifts[lane] = left_shift; + } - return shifts; - }(); + return shifts; + }(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" __always_inline static __m256i get_permute() { - return _mm256_load_si256(reinterpret_cast(permute.data())); - } + return _mm256_load_si256(reinterpret_cast(permute.data())); + } __always_inline static T get_shuffle() { - if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shuffle.data())); - } else { - return _mm256_load_si256(reinterpret_cast(shuffle.data())); - } + if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shuffle.data())); + } else { + return _mm256_load_si256(reinterpret_cast(shuffle.data())); } + } __always_inline static T get_shift() { - if constexpr (std::is_same_v) { - return _mm_load_si128(reinterpret_cast(shift.data())); - } else { - return _mm256_load_si256(reinterpret_cast(shift.data())); - } + if constexpr (std::is_same_v) { + return _mm_load_si128(reinterpret_cast(shift.data())); + } else { + return _mm256_load_si256(reinterpret_cast(shift.data())); } + } #pragma GCC diagnostic pop - }; +}; } // namespace pernix::internal #endif // PERNIX_AVX2_TABLES_H diff --git a/src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h b/src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h index e19051e..dab8472 100644 --- a/src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h +++ b/src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h @@ -12,543 +12,604 @@ using namespace pernix::x86::internal; namespace pernix { -namespace internal { -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -static __always_inline __m512i mm512_clamp_signed_epi32(__m512i input) { - constexpr int32_t min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); - constexpr int32_t max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm512_min_epi32(_mm512_max_epi32(input, _mm512_set1_epi32(min_value)), _mm512_set1_epi32(max_value)); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -static __always_inline __m256i mm256_clamp_signed_epi32_avx512(__m256i input) { - constexpr int32_t min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); - constexpr int32_t max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), _mm256_set1_epi32(max_value)); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -static __always_inline __m512i mm512_clamp_signed_epi64(__m512i input) { - constexpr int64_t min_value = BIT_WIDTH == 1 ? 0 : -(int64_t{1} << (BIT_WIDTH - 1)); - constexpr int64_t max_value = BIT_WIDTH == 1 ? 1 : ((int64_t{1} << (BIT_WIDTH - 1)) - 1); - return _mm512_min_epi64(_mm512_max_epi64(input, _mm512_set1_epi64(min_value)), _mm512_set1_epi64(max_value)); -} - -/** - * @brief Quantize sixteen float values to 32-bit integers. - */ - -static __always_inline __m512i mm512_quantize_ps_epi32(const __m512& input, const __m512& scale) { - const __m512 scaled = _mm512_mul_ps(input, scale); - return _mm512_cvtps_epi32(scaled); -} - -static __always_inline __m512i mm512_quantize_pd_epi64(const __m512d& input, const __m512d& scale) { - const __m512d scaled = _mm512_mul_pd(input, scale); - return _mm512_cvtpd_epi64(scaled); -} - -static __always_inline __m256i mm512_quantize_pd_epi32(const __m512d& input, const __m512d& scale) { - const __m512d scaled = _mm512_mul_pd(input, scale); - return _mm512_cvtpd_epi32(scaled); -} - -static __always_inline __m512i make_m512i_from_2x256(const __m256i a, const __m256i b) { - __m512i result = _mm512_castsi256_si512(a); - result = _mm512_inserti64x4(result, b, 1); - return result; -} - -static __always_inline __m512i make_m512i_from_4x128(const __m128i a, const __m128i b, const __m128i c, const __m128i d) { - __m512i result = _mm512_castsi128_si512(a); - result = _mm512_inserti64x2(result, b, 1); - result = _mm512_inserti64x2(result, c, 2); - result = _mm512_inserti64x2(result, d, 3); - return result; -} - -static __always_inline __m512i make_m512i_from_8x64(const __m128i a, const __m128i b, const __m128i c, const __m128i d, const __m128i e, - const __m128i f, const __m128i g, const __m128i h) { - const __m128i ab = _mm_unpacklo_epi64(a, b); - const __m128i cd = _mm_unpacklo_epi64(c, d); - const __m128i ef = _mm_unpacklo_epi64(e, f); - const __m128i gh = _mm_unpacklo_epi64(g, h); - - __m512i x = _mm512_castsi128_si512(ab); - x = _mm512_inserti32x4(x, cd, 1); - x = _mm512_inserti32x4(x, ef, 2); - x = _mm512_inserti32x4(x, gh, 3); - return x; -} - -static __always_inline __m256i make_m256i_from_2x128(const __m128i a, const __m128i b) { - __m256i result = _mm256_castsi128_si256(a); - result = _mm256_inserti128_si256(result, b, 1); - return result; -} - -static __always_inline __m256i make_m256i_from_4x64(const __m128i a, const __m128i b, const __m128i c, const __m128i d) { - const __m128i ab = _mm_unpacklo_epi64(a, b); - const __m128i cd = _mm_unpacklo_epi64(c, d); - - __m256i x = _mm256_castsi128_si256(ab); - x = _mm256_inserti128_si256(x, cd, 1); - return x; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - constexpr uint32_t iterations_64 = elements_per_block / 64; - constexpr uint32_t iterations_32 = (elements_per_block % 64) / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; - - const __m512 scale_v = _mm512_set1_ps(scale); - - if constexpr (iterations_64 > 0) { -#pragma GCC unroll 8 - for (uint32_t iter = 0; iter < iterations_64; ++iter) { - const __m512 source1 = _mm512_loadu_ps(input); - const __m512 source2 = _mm512_loadu_ps(input + 16); - const __m512 source3 = _mm512_loadu_ps(input + 32); - const __m512 source4 = _mm512_loadu_ps(input + 48); + namespace internal { + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + static __always_inline __m512i mm512_clamp_signed_epi32(__m512i input) { + constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); + return _mm512_min_epi32(_mm512_max_epi32(input, _mm512_set1_epi32(min_value)), + _mm512_set1_epi32(max_value)); + } - const __m512i quantized1 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source4, scale_v)); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + static __always_inline __m256i mm256_clamp_signed_epi32_avx512(__m256i input) { + constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); + return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), + _mm256_set1_epi32(max_value)); + } - const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); - const __m128i converted3 = _mm512_cvtepi32_epi8(quantized3); - const __m128i converted4 = _mm512_cvtepi32_epi8(quantized4); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + static __always_inline __m512i mm512_clamp_signed_epi64(__m512i input) { + constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); + constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); + return _mm512_min_epi64(_mm512_max_epi64(input, _mm512_set1_epi64(min_value)), + _mm512_set1_epi64(max_value)); + } - const __m512i packed = - m512::mm512_pack_epi8_avx512vbmi_1to8(make_m512i_from_4x128(converted1, converted2, converted3, converted4)); + /** + * @brief Quantize sixteen float values to 32-bit integers. + */ - mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); + static __always_inline __m512i mm512_quantize_ps_epi32(const __m512 &input, const __m512 &scale) { + const __m512 scaled = _mm512_mul_ps(input, scale); + return _mm512_cvtps_epi32(scaled); + } - input += 64; - output += 8 * BIT_WIDTH; + static __always_inline __m512i mm512_quantize_pd_epi64(const __m512d &input, const __m512d &scale) { + const __m512d scaled = _mm512_mul_pd(input, scale); + return _mm512_cvtpd_epi64(scaled); } - } - if constexpr (iterations_32 > 0) { - const __m512 source1 = _mm512_loadu_ps(input); - const __m512 source2 = _mm512_loadu_ps(input + 16); + static __always_inline __m256i mm512_quantize_pd_epi32(const __m512d &input, const __m512d &scale) { + const __m512d scaled = _mm512_mul_pd(input, scale); + return _mm512_cvtpd_epi32(scaled); + } - const __m512i quantized1 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source2, scale_v)); + static __always_inline __m512i make_m512i_from_2x256(const __m256i a, const __m256i b) { + __m512i result = _mm512_castsi256_si512(a); + result = _mm512_inserti64x4(result, b, 1); + return result; + } - const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); + static __always_inline __m512i make_m512i_from_4x128(const __m128i a, const __m128i b, const __m128i c, + const __m128i d) { + __m512i result = _mm512_castsi128_si512(a); + result = _mm512_inserti64x2(result, b, 1); + result = _mm512_inserti64x2(result, c, 2); + result = _mm512_inserti64x2(result, d, 3); + return result; + } - const __m256i packed = m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_2x128(converted1, converted2)); - mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); + static __always_inline __m512i make_m512i_from_8x64(const __m128i a, const __m128i b, const __m128i c, + const __m128i d, const __m128i e, + const __m128i f, const __m128i g, const __m128i h) { + const __m128i ab = _mm_unpacklo_epi64(a, b); + const __m128i cd = _mm_unpacklo_epi64(c, d); + const __m128i ef = _mm_unpacklo_epi64(e, f); + const __m128i gh = _mm_unpacklo_epi64(g, h); + + __m512i x = _mm512_castsi128_si512(ab); + x = _mm512_inserti32x4(x, cd, 1); + x = _mm512_inserti32x4(x, ef, 2); + x = _mm512_inserti32x4(x, gh, 3); + return x; + } - input += 32; - output += 4 * BIT_WIDTH; - } + static __always_inline __m256i make_m256i_from_2x128(const __m128i a, const __m128i b) { + __m256i result = _mm256_castsi128_si256(a); + result = _mm256_inserti128_si256(result, b, 1); + return result; + } - if constexpr (iterations_16 > 0) { - const __m512 source = _mm512_loadu_ps(input); - const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); - const __m128i converted = _mm512_cvtepi32_epi8(quantized); + static __always_inline __m256i make_m256i_from_4x64(const __m128i a, const __m128i b, const __m128i c, + const __m128i d) { + const __m128i ab = _mm_unpacklo_epi64(a, b); + const __m128i cd = _mm_unpacklo_epi64(c, d); - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(converted); - mm_storeu_elements_epi16(output, BIT_WIDTH, packed); + __m256i x = _mm256_castsi128_si256(ab); + x = _mm256_inserti128_si256(x, cd, 1); + return x; + } - input += 16; - output += 2 * BIT_WIDTH; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_1to8(const f32 * __restrict__ input, const f32 scale, + u8 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - if constexpr (remaining_elements > 0) { - const __m512 source = mm512_loadu_elements_ps(remaining_elements, input); - const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); - const __m128i converted = _mm512_cvtepi32_epi8(quantized); - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(converted); + constexpr u32 iterations_64 = elements_per_block / 64; + constexpr u32 iterations_32 = (elements_per_block % 64) / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 remaining_elements = + elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; - mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } + const __m512 scale_v = _mm512_set1_ps(scale); - return 0; -} + if constexpr (iterations_64 > 0) { +#pragma GCC unroll 8 + for (u32 iter = 0; iter < iterations_64; ++iter) { + const __m512 source1 = _mm512_loadu_ps(input); + const __m512 source2 = _mm512_loadu_ps(input + 16); + const __m512 source3 = _mm512_loadu_ps(input + 32); + const __m512 source4 = _mm512_loadu_ps(input + 48); + + const __m512i quantized1 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source4, scale_v)); + + const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); + const __m128i converted3 = _mm512_cvtepi32_epi8(quantized3); + const __m128i converted4 = _mm512_cvtepi32_epi8(quantized4); + + const __m512i packed = + m512::mm512_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (make_m512i_from_4x128( + converted1, converted2, converted3, converted4)); + + mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); + + input += 64; + output += 8 * BIT_WIDTH; + } + } + + if constexpr (iterations_32 > 0) { + const __m512 source1 = _mm512_loadu_ps(input); + const __m512 source2 = _mm512_loadu_ps(input + 16); + + const __m512i quantized1 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source2, scale_v)); + + const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); + + const __m256i packed = m256::mm256_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (make_m256i_from_2x128( + converted1, converted2)); + mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); + + input += 32; + output += 4 * BIT_WIDTH; + } + + if constexpr (iterations_16 > 0) { + const __m512 source = _mm512_loadu_ps(input); + const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); + const __m128i converted = _mm512_cvtepi32_epi8(quantized); + + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (converted); + mm_storeu_elements_epi16(output, BIT_WIDTH, packed); + + input += 16; + output += 2 * BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const __m512 source = mm512_loadu_elements_ps(remaining_elements, input); + const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); + const __m128i converted = _mm512_cvtepi32_epi8(quantized); + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (converted); + + mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } + + return 0; + } -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_9to16(const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_9to16(const f32 * __restrict__ input, const f32 scale, + u8 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_32 = elements_per_block / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = + elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; - const __m512 scale_v = _mm512_set1_ps(scale); - const __m256 scale_v256 = _mm256_set1_ps(scale); + const __m512 scale_v = _mm512_set1_ps(scale); + const __m256 scale_v256 = _mm256_set1_ps(scale); - if constexpr (iterations_32 > 0) { + if constexpr (iterations_32 > 0) { #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_32; ++iter) { - const __m512 source1 = _mm512_loadu_ps(input); - const __m512 source2 = _mm512_loadu_ps(input + 16); - - const __m512i quantized1 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source2, scale_v)); - - const __m256i converted1 = _mm512_cvtepi32_epi16(quantized1); - const __m256i converted2 = _mm512_cvtepi32_epi16(quantized2); - - const __m512i packed = m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_2x256(converted1, converted2)); - mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - - input += 32; - output += 4 * BIT_WIDTH; + for (u32 iter = 0; iter < iterations_32; ++iter) { + const __m512 source1 = _mm512_loadu_ps(input); + const __m512 source2 = _mm512_loadu_ps(input + 16); + + const __m512i quantized1 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source2, scale_v)); + + const __m256i converted1 = _mm512_cvtepi32_epi16(quantized1); + const __m256i converted2 = _mm512_cvtepi32_epi16(quantized2); + + const __m512i packed = m512::mm512_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (make_m512i_from_2x256( + converted1, converted2)); + mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); + + input += 32; + output += 4 * BIT_WIDTH; + } + } + + if constexpr (iterations_16 > 0) { + const __m512 source = _mm512_loadu_ps(input); + const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); + const __m256i converted = _mm512_cvtepi32_epi16(quantized); + + const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); + mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); + + input += 16; + output += 2 * BIT_WIDTH; + } + + if constexpr (iterations_8 > 0) { + const __m256 source = _mm256_loadu_ps(input); + const __m256i quantized = mm256_clamp_signed_epi32_avx512( + mm256_quantize_ps_epi32(source, scale_v256)); + const __m128i converted = _mm256_cvtepi32_epi16(quantized); + + const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); + mm_storeu_elements_epi8(output, BIT_WIDTH, packed); + + input += 8; + output += BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); + const __m256i quantized = mm256_clamp_signed_epi32_avx512( + mm256_quantize_ps_epi32(source, scale_v256)); + const __m128i converted = _mm256_cvtepi32_epi16(quantized); + const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); + + mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } + + return 0; } - } - if constexpr (iterations_16 > 0) { - const __m512 source = _mm512_loadu_ps(input); - const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); - const __m256i converted = _mm512_cvtepi32_epi16(quantized); + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_17to24(const f32 * __restrict__ input, const f32 scale, + u8 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(converted); - mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; - input += 16; - output += 2 * BIT_WIDTH; - } + const __m512 scale_v = _mm512_set1_ps(scale); + const __m256 scale_v256 = _mm256_set1_ps(scale); - if constexpr (iterations_8 > 0) { - const __m256 source = _mm256_loadu_ps(input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); - const __m128i converted = _mm256_cvtepi32_epi16(quantized); - - const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); - mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - - input += 8; - output += BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); - const __m128i converted = _mm256_cvtepi32_epi16(quantized); - const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); - - mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_17to24(const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - constexpr uint32_t iterations_16 = elements_per_block / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; - - const __m512 scale_v = _mm512_set1_ps(scale); - const __m256 scale_v256 = _mm256_set1_ps(scale); - - if constexpr (iterations_16 > 0) { + if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (uint32_t i = 0; i < iterations_16; ++i) { - const __m512 source = _mm512_loadu_ps(input); - const __m512i packed_input = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); - - const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(packed_input); - mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; - output += 2 * BIT_WIDTH; + for (u32 i = 0; i < iterations_16; ++i) { + const __m512 source = _mm512_loadu_ps(input); + const __m512i packed_input = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source, scale_v)); + + const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (packed_input); + mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); + input += 16; + output += 2 * BIT_WIDTH; + } + } + + if constexpr (iterations_8 > 0) { + const __m256 source = _mm256_loadu_ps(input); + const __m256i packed_input = mm256_clamp_signed_epi32_avx512( + mm256_quantize_ps_epi32(source, scale_v256)); + + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (packed_input); + mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); + + input += 8; + output += BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); + const __m256i packed_input = mm256_clamp_signed_epi32_avx512( + mm256_quantize_ps_epi32(source, scale_v256)); + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (packed_input); + + mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } + + return 0; } - } - - if constexpr (iterations_8 > 0) { - const __m256 source = _mm256_loadu_ps(input); - const __m256i packed_input = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); - mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_1to8(const f64 * __restrict__ input, const f64 scale, + u8 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - input += 8; - output += BIT_WIDTH; - } + constexpr u32 iterations_64 = elements_per_block / 64; + constexpr u32 iterations_32 = (elements_per_block % 64) / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 remaining_elements = + elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; - if constexpr (remaining_elements > 0) { - const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); - const __m256i packed_input = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); + const __m512d scale_v = _mm512_set1_pd(scale); - mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_1to8(const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - constexpr uint32_t iterations_64 = elements_per_block / 64; - constexpr uint32_t iterations_32 = (elements_per_block % 64) / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; - - const __m512d scale_v = _mm512_set1_pd(scale); - - if constexpr (iterations_64 > 0) { + if constexpr (iterations_64 > 0) { #pragma GCC unroll 8 - for (uint32_t iter = 0; iter < iterations_64; ++iter) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512d source3 = _mm512_loadu_pd(input + 16); - const __m512d source4 = _mm512_loadu_pd(input + 24); - const __m512d source5 = _mm512_loadu_pd(input + 32); - const __m512d source6 = _mm512_loadu_pd(input + 40); - const __m512d source7 = _mm512_loadu_pd(input + 48); - const __m512d source8 = _mm512_loadu_pd(input + 56); - - const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source4, scale_v)); - const __m512i quantized5 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source5, scale_v)); - const __m512i quantized6 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source6, scale_v)); - const __m512i quantized7 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source7, scale_v)); - const __m512i quantized8 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source8, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - const __m128i converted3 = _mm512_cvtepi64_epi8(quantized3); - const __m128i converted4 = _mm512_cvtepi64_epi8(quantized4); - const __m128i converted5 = _mm512_cvtepi64_epi8(quantized5); - const __m128i converted6 = _mm512_cvtepi64_epi8(quantized6); - const __m128i converted7 = _mm512_cvtepi64_epi8(quantized7); - const __m128i converted8 = _mm512_cvtepi64_epi8(quantized8); - - const __m512i packed = m512::mm512_pack_epi8_avx512vbmi_1to8( - make_m512i_from_8x64(converted1, converted2, converted3, converted4, converted5, converted6, converted7, converted8)); - - mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - - input += 64; - output += 8 * BIT_WIDTH; + for (u32 iter = 0; iter < iterations_64; ++iter) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512d source3 = _mm512_loadu_pd(input + 16); + const __m512d source4 = _mm512_loadu_pd(input + 24); + const __m512d source5 = _mm512_loadu_pd(input + 32); + const __m512d source6 = _mm512_loadu_pd(input + 40); + const __m512d source7 = _mm512_loadu_pd(input + 48); + const __m512d source8 = _mm512_loadu_pd(input + 56); + + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source4, scale_v)); + const __m512i quantized5 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source5, scale_v)); + const __m512i quantized6 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source6, scale_v)); + const __m512i quantized7 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source7, scale_v)); + const __m512i quantized8 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source8, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); + const __m128i converted3 = _mm512_cvtepi64_epi8(quantized3); + const __m128i converted4 = _mm512_cvtepi64_epi8(quantized4); + const __m128i converted5 = _mm512_cvtepi64_epi8(quantized5); + const __m128i converted6 = _mm512_cvtepi64_epi8(quantized6); + const __m128i converted7 = _mm512_cvtepi64_epi8(quantized7); + const __m128i converted8 = _mm512_cvtepi64_epi8(quantized8); + + const __m512i packed = m512::mm512_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > ( + make_m512i_from_8x64(converted1, converted2, converted3, converted4, + converted5, converted6, converted7, converted8)); + + mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); + + input += 64; + output += 8 * BIT_WIDTH; + } + } + + if constexpr (iterations_32 > 0) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512d source3 = _mm512_loadu_pd(input + 16); + const __m512d source4 = _mm512_loadu_pd(input + 24); + + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source4, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); + const __m128i converted3 = _mm512_cvtepi64_epi8(quantized3); + const __m128i converted4 = _mm512_cvtepi64_epi8(quantized4); + + const __m256i packed = + m256::mm256_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (make_m256i_from_4x64( + converted1, converted2, converted3, converted4)); + + mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); + + input += 32; + output += 4 * BIT_WIDTH; + } + + if constexpr (iterations_16 > 0) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); + + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (_mm_unpacklo_epi64( + converted1, converted2)); + + mm_storeu_elements_epi16(output, BIT_WIDTH, packed); + + input += 16; + output += 2 * BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + constexpr u32 source1_elements = remaining_elements > 8 ? 8 : remaining_elements; + constexpr u32 source2_elements = remaining_elements > 8 ? remaining_elements - 8 : 0; + + const __m512d source1 = mm512_loadu_elements_pd(source1_elements, input); + const __m512d source2 = source2_elements > 0 + ? mm512_loadu_elements_pd(source2_elements, input + 8) + : _mm512_setzero_pd(); + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); + + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (_mm_unpacklo_epi64( + converted1, converted2)); + + mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } + + return 0; } - } - if constexpr (iterations_32 > 0) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512d source3 = _mm512_loadu_pd(input + 16); - const __m512d source4 = _mm512_loadu_pd(input + 24); - - const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source4, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - const __m128i converted3 = _mm512_cvtepi64_epi8(quantized3); - const __m128i converted4 = _mm512_cvtepi64_epi8(quantized4); - - const __m256i packed = - m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_4x64(converted1, converted2, converted3, converted4)); - - mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - - input += 32; - output += 4 * BIT_WIDTH; - } + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_9to16(const f64 * __restrict__ input, const f64 scale, + u8 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - if constexpr (iterations_16 > 0) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = + elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; - const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); + const __m512d scale_v = _mm512_set1_pd(scale); - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(_mm_unpacklo_epi64(converted1, converted2)); - - mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - - input += 16; - output += 2 * BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - constexpr uint32_t source1_elements = remaining_elements > 8 ? 8 : remaining_elements; - constexpr uint32_t source2_elements = remaining_elements > 8 ? remaining_elements - 8 : 0; - - const __m512d source1 = mm512_loadu_elements_pd(source1_elements, input); - const __m512d source2 = source2_elements > 0 ? mm512_loadu_elements_pd(source2_elements, input + 8) : _mm512_setzero_pd(); - const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(_mm_unpacklo_epi64(converted1, converted2)); - - mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; -} - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_9to16(const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - constexpr uint32_t iterations_32 = elements_per_block / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; - - const __m512d scale_v = _mm512_set1_pd(scale); - - if constexpr (iterations_32 > 0) { + if constexpr (iterations_32 > 0) { #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_32; ++iter) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512d source3 = _mm512_loadu_pd(input + 16); - const __m512d source4 = _mm512_loadu_pd(input + 24); - - const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source4, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); - const __m128i converted3 = _mm512_cvtepi64_epi16(quantized3); - const __m128i converted4 = _mm512_cvtepi64_epi16(quantized4); - - const __m512i packed = - m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_4x128(converted1, converted2, converted3, converted4)); - - mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - - input += 32; - output += 4 * BIT_WIDTH; + for (u32 iter = 0; iter < iterations_32; ++iter) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512d source3 = _mm512_loadu_pd(input + 16); + const __m512d source4 = _mm512_loadu_pd(input + 24); + + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source4, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); + const __m128i converted3 = _mm512_cvtepi64_epi16(quantized3); + const __m128i converted4 = _mm512_cvtepi64_epi16(quantized4); + + const __m512i packed = + m512::mm512_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (make_m512i_from_4x128( + converted1, converted2, converted3, converted4)); + + mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); + + input += 32; + output += 4 * BIT_WIDTH; + } + } + + if constexpr (iterations_16 > 0) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); + + const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (make_m256i_from_2x128( + converted1, converted2)); + + mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); + + input += 16; + output += 2 * BIT_WIDTH; + } + + if constexpr (iterations_8 > 0) { + const __m512d source = _mm512_loadu_pd(input); + const __m512i quantized = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source, scale_v)); + const __m128i converted = _mm512_cvtepi64_epi16(quantized); + + const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); + mm_storeu_elements_epi8(output, BIT_WIDTH, packed); + + input += 8; + output += BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const __m512d source = mm512_loadu_elements_pd(remaining_elements, input); + const __m512i quantized = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source, scale_v)); + const __m128i converted = _mm512_cvtepi64_epi16(quantized); + + const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); + mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } + + return 0; } - } - - if constexpr (iterations_16 > 0) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); - - const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(make_m256i_from_2x128(converted1, converted2)); - - mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - - input += 16; - output += 2 * BIT_WIDTH; - } - - if constexpr (iterations_8 > 0) { - const __m512d source = _mm512_loadu_pd(input); - const __m512i quantized = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source, scale_v)); - const __m128i converted = _mm512_cvtepi64_epi16(quantized); - const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); - mm_storeu_elements_epi8(output, BIT_WIDTH, packed); + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_17to24(const f64 * __restrict__ input, const f64 scale, + u8 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - input += 8; - output += BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - const __m512d source = mm512_loadu_elements_pd(remaining_elements, input); - const __m512i quantized = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source, scale_v)); - const __m128i converted = _mm512_cvtepi64_epi16(quantized); - - const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); - mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; -} - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_17to24(const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - constexpr uint32_t iterations_16 = elements_per_block / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; - const __m512d scale_v = _mm512_set1_pd(scale); + const __m512d scale_v = _mm512_set1_pd(scale); - if constexpr (iterations_16 > 0) { + if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (uint32_t i = 0; i < iterations_16; ++i) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - - const __m256i quantized1 = mm256_clamp_signed_epi32_avx512(mm512_quantize_pd_epi32(source1, scale_v)); - const __m256i quantized2 = mm256_clamp_signed_epi32_avx512(mm512_quantize_pd_epi32(source2, scale_v)); - - const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(make_m512i_from_2x256(quantized1, quantized2)); - mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - - input += 16; - output += 2 * BIT_WIDTH; + for (u32 i = 0; i < iterations_16; ++i) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + + const __m256i quantized1 = mm256_clamp_signed_epi32_avx512( + mm512_quantize_pd_epi32(source1, scale_v)); + const __m256i quantized2 = mm256_clamp_signed_epi32_avx512( + mm512_quantize_pd_epi32(source2, scale_v)); + + const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > ( + make_m512i_from_2x256(quantized1, quantized2)); + mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); + + input += 16; + output += 2 * BIT_WIDTH; + } + } + + if constexpr (iterations_8 > 0) { + const __m512d source = _mm512_loadu_pd(input); + const __m256i quantized = mm256_clamp_signed_epi32_avx512( + mm512_quantize_pd_epi32(source, scale_v)); + + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (quantized); + mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); + + input += 8; + output += BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const __m512d source = mm512_loadu_elements_pd(remaining_elements, input); + const __m256i quantized = mm256_clamp_signed_epi32_avx512( + mm512_quantize_pd_epi32(source, scale_v)); + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (quantized); + + mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } + + return 0; } - } + } // namespace internal - if constexpr (iterations_8 > 0) { - const __m512d source = _mm512_loadu_pd(input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512(mm512_quantize_pd_epi32(source, scale_v)); - - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); - mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - - input += 8; - output += BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - const __m512d source = mm512_loadu_elements_pd(remaining_elements, input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512(mm512_quantize_pd_epi32(source, scale_v)); - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); - - mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; -} -} // namespace internal - -/** + /** * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -560,24 +621,25 @@ __always_inline int mm512_compress_block_avx512vbmi_17to24(const double_t* __res * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - std::memset(output, 0, BLOCK_SIZE); - - if constexpr (BIT_WIDTH <= 8) { - return internal::mm512_compress_block_avx512vbmi_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH <= 16) { - return internal::mm512_compress_block_avx512vbmi_9to16(input, scale, output); - } else { - return internal::mm512_compress_block_avx512vbmi_17to24(input, scale, output); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm512_compress_block_avx512vbmi(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + std::memset(output, 0, BLOCK_SIZE); + + if constexpr (BIT_WIDTH <= 8) { + return internal::mm512_compress_block_avx512vbmi_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH <= 16) { + return internal::mm512_compress_block_avx512vbmi_9to16(input, scale, output); + } else { + return internal::mm512_compress_block_avx512vbmi_17to24(input, scale, output); + } } -} -/** + /** * @brief Compress a single block of double values using AVX-512 and AVX-512-VBMI instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -588,24 +650,25 @@ int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const fl * * @note This overload is declared for parity with the float path. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - std::memset(output, 0, BLOCK_SIZE); - - if constexpr (BIT_WIDTH <= 8) { - return internal::mm512_compress_block_avx512vbmi_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH <= 16) { - return internal::mm512_compress_block_avx512vbmi_9to16(input, scale, output); - } else { - return internal::mm512_compress_block_avx512vbmi_17to24(input, scale, output); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm512_compress_block_avx512vbmi(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + std::memset(output, 0, BLOCK_SIZE); + + if constexpr (BIT_WIDTH <= 8) { + return internal::mm512_compress_block_avx512vbmi_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH <= 16) { + return internal::mm512_compress_block_avx512vbmi_9to16(input, scale, output); + } else { + return internal::mm512_compress_block_avx512vbmi_17to24(input, scale, output); + } } -} -/** + /** * @brief Compress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -618,26 +681,27 @@ int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const do * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const float_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm512_compress_blocks_avx512vbmi(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const f32 *block_input = input; + u8 *block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + mm512_compress_block_avx512vbmi(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } - return 0; -} + return 0; + } -/** + /** * @brief Compress multiple blocks of double values using AVX-512 and AVX-512-VBMI instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -647,24 +711,25 @@ int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f * @param blocks number of blocks to compress. * @return int status code. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const double_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm512_compress_blocks_avx512vbmi(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const f64 *block_input = input; + u8 *block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + mm512_compress_block_avx512vbmi(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } - return 0; -} + return 0; + } } // namespace pernix #endif // PERNIX_AVX512VBMI_COMPRESSION_H diff --git a/src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h b/src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h index d0f98f0..a004be5 100644 --- a/src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h +++ b/src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h @@ -12,493 +12,541 @@ using namespace pernix::x86::internal; namespace pernix { -namespace internal { -/** + namespace internal { + /** * @brief Dequantize sixteen integer values to floats. */ -__always_inline __m512 mm512_dequantize_epi32(const __m512i& input, const __m512& scale) { - const __m512 converted = _mm512_cvtepi32_ps(input); - return _mm512_mul_ps(converted, scale); -} - -__always_inline __m512d mm512_dequantize_epi64(const __m512i& input, const __m512d& scale) { - const __m512d converted = _mm512_cvtepi64_pd(input); - return _mm512_mul_pd(converted, scale); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - const uint32_t iterations_64 = elements_per_block / 64; - const uint32_t iterations_32 = (elements_per_block % 64) / 32; - const uint32_t iterations_16 = (elements_per_block % 32) / 16; - const uint32_t remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; - - const __m512 scale_v = _mm512_set1_ps(scale); - - if constexpr (iterations_64 > 0) { -#pragma GCC unroll 8 - for (uint32_t i = 0; i < iterations_64; ++i) { - const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8(source); - - const __m512i converted1 = _mm512_cvtepi8_epi32(_mm512_castsi512_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 1)); - const __m512i converted3 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 2)); - const __m512i converted4 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 3)); - - const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); - const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); - const __m512 dequantized3 = mm512_dequantize_epi32(converted3, scale_v); - const __m512 dequantized4 = mm512_dequantize_epi32(converted4, scale_v); - - _mm512_storeu_ps(output, dequantized1); - _mm512_storeu_ps(output + 16, dequantized2); - _mm512_storeu_ps(output + 32, dequantized3); - _mm512_storeu_ps(output + 48, dequantized4); - - output += 64; - input += 8 * BIT_WIDTH; +__always_inline __m512 mm512_dequantize_epi32(const __m512i &input, const __m512 &scale) { + const __m512 converted = _mm512_cvtepi32_ps(input); + return _mm512_mul_ps(converted, scale); } - } - - if constexpr (iterations_32 > 0) { - const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8(source); - - const __m512i converted1 = _mm512_cvtepi8_epi32(_mm256_castsi256_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi8_epi32(_mm256_extracti128_si256(unpacked, 1)); - - const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); - const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); - - _mm512_storeu_ps(output, dequantized1); - _mm512_storeu_ps(output + 16, dequantized2); - - output += 32; - input += 4 * BIT_WIDTH; - } - - if constexpr (iterations_16 > 0) { - const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted = _mm512_cvtepi8_epi32(unpacked); - - const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); - - _mm512_storeu_ps(output, dequantized); - - output += 16; - input += 2 * BIT_WIDTH; - } +__always_inline __m512d mm512_dequantize_epi64(const __m512i &input, const __m512d &scale) { + const __m512d converted = _mm512_cvtepi64_pd(input); + return _mm512_mul_pd(converted, scale); + } - if constexpr (remaining_elements > 0) { - const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8 * __restrict__ input, const f32 scale, + f32 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const __m512i converted = _mm512_cvtepi8_epi32(unpacked); + const u32 iterations_64 = elements_per_block / 64; + const u32 iterations_32 = (elements_per_block % 64) / 32; + const u32 iterations_16 = (elements_per_block % 32) / 16; + const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - + iterations_16 * 16; - const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); + const __m512 scale_v = _mm512_set1_ps(scale); - mm512_storeu_elements_ps(output, remaining_elements, dequantized); - } + if constexpr (iterations_64 > 0) { +#pragma GCC unroll 8 + for (u32 i = 0; i < iterations_64; ++i) { + const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES + > + (source); + + const __m512i converted1 = _mm512_cvtepi8_epi32(_mm512_castsi512_si128(unpacked)); + const __m512i converted2 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 1)); + const __m512i converted3 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 2)); + const __m512i converted4 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 3)); + + const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); + const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); + const __m512 dequantized3 = mm512_dequantize_epi32(converted3, scale_v); + const __m512 dequantized4 = mm512_dequantize_epi32(converted4, scale_v); + + _mm512_storeu_ps(output, dequantized1); + _mm512_storeu_ps(output + 16, dequantized2); + _mm512_storeu_ps(output + 32, dequantized3); + _mm512_storeu_ps(output + 48, dequantized4); + + output += 64; + input += 8 * BIT_WIDTH; + } + } - return 0; -} + if constexpr (iterations_32 > 0) { + const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES + > + (source); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + const __m512i converted1 = _mm512_cvtepi8_epi32(_mm256_castsi256_si128(unpacked)); + const __m512i converted2 = _mm512_cvtepi8_epi32(_mm256_extracti128_si256(unpacked, 1)); - const uint32_t iterations_64 = elements_per_block / 64; - const uint32_t iterations_32 = (elements_per_block % 64) / 32; - const uint32_t iterations_16 = (elements_per_block % 32) / 16; - const uint32_t remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); + const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); - const __m512d scale_v = _mm512_set1_pd(scale); + _mm512_storeu_ps(output, dequantized1); + _mm512_storeu_ps(output + 16, dequantized2); - if constexpr (iterations_64 > 0) { -#pragma GCC unroll 8 - for (uint32_t i = 0; i < iterations_64; ++i) { - const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8(source); - - const __m128i extracted1 = _mm512_castsi512_si128(unpacked); - const __m128i extracted2 = _mm512_extracti64x2_epi64(unpacked, 1); - const __m128i extracted3 = _mm512_extracti64x2_epi64(unpacked, 2); - const __m128i extracted4 = _mm512_extracti64x2_epi64(unpacked, 3); - - const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); - const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); - const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); - const __m512i converted5 = _mm512_cvtepi8_epi64(extracted3); - const __m512i converted6 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted3, 8)); - const __m512i converted7 = _mm512_cvtepi8_epi64(extracted4); - const __m512i converted8 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted4, 8)); - - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); - const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); - const __m512d dequantized5 = mm512_dequantize_epi64(converted5, scale_v); - const __m512d dequantized6 = mm512_dequantize_epi64(converted6, scale_v); - const __m512d dequantized7 = mm512_dequantize_epi64(converted7, scale_v); - const __m512d dequantized8 = mm512_dequantize_epi64(converted8, scale_v); - - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); - _mm512_storeu_pd(output + 16, dequantized3); - _mm512_storeu_pd(output + 24, dequantized4); - _mm512_storeu_pd(output + 32, dequantized5); - _mm512_storeu_pd(output + 40, dequantized6); - _mm512_storeu_pd(output + 48, dequantized7); - _mm512_storeu_pd(output + 56, dequantized8); - - output += 64; - input += 8 * BIT_WIDTH; - } - - if constexpr (iterations_32 > 0) { - const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8(source); + output += 32; + input += 4 * BIT_WIDTH; + } - const __m128i extracted1 = _mm256_castsi256_si128(unpacked); - const __m128i extracted2 = _mm256_extracti64x2_epi64(unpacked, 1); + if constexpr (iterations_16 > 0) { + const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); - const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); - const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); + const __m512i converted = _mm512_cvtepi8_epi32(unpacked); - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); - const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); + const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); - _mm512_storeu_pd(output + 16, dequantized3); - _mm512_storeu_pd(output + 24, dequantized4); + _mm512_storeu_ps(output, dequantized); - output += 32; - input += 4 * BIT_WIDTH; - } + output += 16; + input += 2 * BIT_WIDTH; + } - if constexpr (iterations_16 > 0) { - const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); + if constexpr (remaining_elements > 0) { + const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); + const __m512i converted = _mm512_cvtepi8_epi32(unpacked); - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); + mm512_storeu_elements_ps(output, remaining_elements, dequantized); + } - output += 16; - input += 2 * BIT_WIDTH; + return 0; } - if constexpr (remaining_elements > 0) { - const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8 * __restrict__ input, const f64 scale, + f64 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); + const u32 iterations_64 = elements_per_block / 64; + const u32 iterations_32 = (elements_per_block % 64) / 32; + const u32 iterations_16 = (elements_per_block % 32) / 16; + const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - + iterations_16 * 16; - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512d scale_v = _mm512_set1_pd(scale); - mm512_storeu_elements_pd(output, remaining_elements < 8 ? remaining_elements : 8, dequantized1); - if constexpr (remaining_elements > 8) { - mm512_storeu_elements_pd(output + 8, remaining_elements - 8, dequantized2); + if constexpr (iterations_64 > 0) { +#pragma GCC unroll 8 + for (u32 i = 0; i < iterations_64; ++i) { + const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES + > + (source); + + const __m128i extracted1 = _mm512_castsi512_si128(unpacked); + const __m128i extracted2 = _mm512_extracti64x2_epi64(unpacked, 1); + const __m128i extracted3 = _mm512_extracti64x2_epi64(unpacked, 2); + const __m128i extracted4 = _mm512_extracti64x2_epi64(unpacked, 3); + + const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); + const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); + const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); + const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); + const __m512i converted5 = _mm512_cvtepi8_epi64(extracted3); + const __m512i converted6 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted3, 8)); + const __m512i converted7 = _mm512_cvtepi8_epi64(extracted4); + const __m512i converted8 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted4, 8)); + + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); + const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); + const __m512d dequantized5 = mm512_dequantize_epi64(converted5, scale_v); + const __m512d dequantized6 = mm512_dequantize_epi64(converted6, scale_v); + const __m512d dequantized7 = mm512_dequantize_epi64(converted7, scale_v); + const __m512d dequantized8 = mm512_dequantize_epi64(converted8, scale_v); + + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); + _mm512_storeu_pd(output + 16, dequantized3); + _mm512_storeu_pd(output + 24, dequantized4); + _mm512_storeu_pd(output + 32, dequantized5); + _mm512_storeu_pd(output + 40, dequantized6); + _mm512_storeu_pd(output + 48, dequantized7); + _mm512_storeu_pd(output + 56, dequantized8); + + output += 64; + input += 8 * BIT_WIDTH; + } + + if constexpr (iterations_32 > 0) { + const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES + > + (source); + + const __m128i extracted1 = _mm256_castsi256_si128(unpacked); + const __m128i extracted2 = _mm256_extracti64x2_epi64(unpacked, 1); + + const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); + const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); + const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); + const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); + + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); + const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); + + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); + _mm512_storeu_pd(output + 16, dequantized3); + _mm512_storeu_pd(output + 24, dequantized4); + + output += 32; + input += 4 * BIT_WIDTH; + } + + if constexpr (iterations_16 > 0) { + const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES + > + (source); + + const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); + const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); + + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); + + output += 16; + input += 2 * BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES + > + (source); + + const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); + const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); + + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + + mm512_storeu_elements_pd(output, remaining_elements < 8 ? remaining_elements : 8, dequantized1); + if constexpr (remaining_elements > 8) { + mm512_storeu_elements_pd(output + 8, remaining_elements - 8, dequantized2); + } + } } - } - } - return 0; -} + return 0; + } -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8 * __restrict__ input, const f32 scale, + f32 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_32 = elements_per_block / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = + elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; - const __m512 scale_v = _mm512_set1_ps(scale); - const __m256 scale_v256 = _mm256_set1_ps(scale); + const __m512 scale_v = _mm512_set1_ps(scale); + const __m256 scale_v256 = _mm256_set1_ps(scale); - if constexpr (iterations_32 > 0) { + if constexpr (iterations_32 > 0) { #pragma GCC unroll 4 - for (uint32_t i = 0; i < iterations_32; ++i) { - const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16(source); + for (u32 i = 0; i < iterations_32; ++i) { + const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted1 = _mm512_cvtepi16_epi32(_mm512_castsi512_si256(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi32(_mm512_extracti32x8_epi32(unpacked, 1)); + const __m512i converted1 = _mm512_cvtepi16_epi32(_mm512_castsi512_si256(unpacked)); + const __m512i converted2 = _mm512_cvtepi16_epi32(_mm512_extracti32x8_epi32(unpacked, 1)); - const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); - const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); + const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); + const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); - _mm512_storeu_ps(output, dequantized1); - _mm512_storeu_ps(output + 16, dequantized2); + _mm512_storeu_ps(output, dequantized1); + _mm512_storeu_ps(output + 16, dequantized2); - output += 32; - input += 4 * BIT_WIDTH; - } - } + output += 32; + input += 4 * BIT_WIDTH; + } + } - if constexpr (iterations_16 > 0) { - const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16(source); + if constexpr (iterations_16 > 0) { + const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted = _mm512_cvtepi16_epi32(unpacked); - const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); + const __m512i converted = _mm512_cvtepi16_epi32(unpacked); + const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); - _mm512_storeu_ps(output, dequantized); + _mm512_storeu_ps(output, dequantized); - output += 16; - input += 2 * BIT_WIDTH; - } + output += 16; + input += 2 * BIT_WIDTH; + } - if constexpr (iterations_8 > 0) { - const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); + if constexpr (iterations_8 > 0) { + const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m256i converted = _mm256_cvtepi16_epi32(unpacked); - const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); + const __m256i converted = _mm256_cvtepi16_epi32(unpacked); + const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); - _mm256_storeu_ps(output, dequantized); + _mm256_storeu_ps(output, dequantized); - output += 8; - input += BIT_WIDTH; - } + output += 8; + input += BIT_WIDTH; + } - if constexpr (remaining_elements > 0) { - const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); + if constexpr (remaining_elements > 0) { + const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m256i converted = _mm256_cvtepi16_epi32(unpacked); - const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); + const __m256i converted = _mm256_cvtepi16_epi32(unpacked); + const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); - mm256_storeu_elements_ps(output, remaining_elements, dequantized); - } + mm256_storeu_elements_ps(output, remaining_elements, dequantized); + } - return 0; -} + return 0; + } -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8 * __restrict__ input, const f64 scale, + f64 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_32 = elements_per_block / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = + elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; - const __m512d scale_v = _mm512_set1_pd(scale); + const __m512d scale_v = _mm512_set1_pd(scale); - if constexpr (iterations_32 > 0) { + if constexpr (iterations_32 > 0) { #pragma GCC unroll 4 - for (uint32_t i = 0; i < iterations_32; ++i) { - const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16(source); - - const __m512i converted1 = _mm512_cvtepi16_epi64(_mm512_castsi512_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 1)); - const __m512i converted3 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 2)); - const __m512i converted4 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 3)); - - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); - const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); - - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); - _mm512_storeu_pd(output + 16, dequantized3); - _mm512_storeu_pd(output + 24, dequantized4); - - output += 32; - input += 4 * BIT_WIDTH; - } - } + for (u32 i = 0; i < iterations_32; ++i) { + const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + > + (source); + + const __m512i converted1 = _mm512_cvtepi16_epi64(_mm512_castsi512_si128(unpacked)); + const __m512i converted2 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 1)); + const __m512i converted3 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 2)); + const __m512i converted4 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 3)); + + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); + const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); + + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); + _mm512_storeu_pd(output + 16, dequantized3); + _mm512_storeu_pd(output + 24, dequantized4); + + output += 32; + input += 4 * BIT_WIDTH; + } + } - if constexpr (iterations_16 > 0) { - const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16(source); + if constexpr (iterations_16 > 0) { + const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted1 = _mm512_cvtepi16_epi64(_mm256_castsi256_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi64(_mm256_extracti64x2_epi64(unpacked, 1)); + const __m512i converted1 = _mm512_cvtepi16_epi64(_mm256_castsi256_si128(unpacked)); + const __m512i converted2 = _mm512_cvtepi16_epi64(_mm256_extracti64x2_epi64(unpacked, 1)); - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); - output += 16; - input += 2 * BIT_WIDTH; - } + output += 16; + input += 2 * BIT_WIDTH; + } - if constexpr (iterations_8 > 0) { - const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); + if constexpr (iterations_8 > 0) { + const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted = _mm512_cvtepi16_epi64(unpacked); + const __m512i converted = _mm512_cvtepi16_epi64(unpacked); - const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); - _mm512_storeu_pd(output, dequantized); + _mm512_storeu_pd(output, dequantized); - output += 8; - input += BIT_WIDTH; - } + output += 8; + input += BIT_WIDTH; + } - if constexpr (remaining_elements > 0) { - const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); + if constexpr (remaining_elements > 0) { + const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted = _mm512_cvtepi16_epi64(unpacked); + const __m512i converted = _mm512_cvtepi16_epi64(unpacked); - const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); - mm512_storeu_elements_pd(output, remaining_elements, dequantized); - } + mm512_storeu_elements_pd(output, remaining_elements, dequantized); + } - return 0; -} + return 0; + } -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8 * __restrict__ input, const f32 scale, + f32 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_16 = elements_per_block / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; - const __m512 scale_v = _mm512_set1_ps(scale); - const __m256 scale_v256 = _mm256_set1_ps(scale); + const __m512 scale_v = _mm512_set1_ps(scale); + const __m256 scale_v256 = _mm256_set1_ps(scale); - if constexpr (iterations_16 > 0) { + if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (uint32_t i = 0; i < iterations_16; ++i) { - const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24(source); + for (u32 i = 0; i < iterations_16; ++i) { + const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512 dequantized = mm512_dequantize_epi32(unpacked, scale_v); + const __m512 dequantized = mm512_dequantize_epi32(unpacked, scale_v); - _mm512_storeu_ps(output, dequantized); + _mm512_storeu_ps(output, dequantized); - output += 16; - input += 2 * BIT_WIDTH; - } - } + output += 16; + input += 2 * BIT_WIDTH; + } + } - if constexpr (iterations_8 > 0) { - const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24(source); + if constexpr (iterations_8 > 0) { + const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); + const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); - _mm256_storeu_ps(output, dequantized); + _mm256_storeu_ps(output, dequantized); - output += 8; - input += BIT_WIDTH; - } + output += 8; + input += BIT_WIDTH; + } - if constexpr (remaining_elements > 0) { - const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24(source); + if constexpr (remaining_elements > 0) { + const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); + const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); - mm256_storeu_elements_ps(output, remaining_elements, dequantized); - } + mm256_storeu_elements_ps(output, remaining_elements, dequantized); + } - return 0; -} + return 0; + } -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8 * __restrict__ input, const f64 scale, + f64 * __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_16 = elements_per_block / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; - const __m512d scale_v = _mm512_set1_pd(scale); + const __m512d scale_v = _mm512_set1_pd(scale); - if constexpr (iterations_16 > 0) { + if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (uint32_t i = 0; i < iterations_16; ++i) { - const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24(source); + for (u32 i = 0; i < iterations_16; ++i) { + const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(unpacked)); - const __m512i converted2 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64(unpacked, 1)); + const __m512i converted1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(unpacked)); + const __m512i converted2 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64(unpacked, 1)); - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); - output += 16; - input += 2 * BIT_WIDTH; - } - } + output += 16; + input += 2 * BIT_WIDTH; + } + } - if constexpr (iterations_8 > 0) { - const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24(source); + if constexpr (iterations_8 > 0) { + const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted = _mm512_cvtepi32_epi64(unpacked); + const __m512i converted = _mm512_cvtepi32_epi64(unpacked); - const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); - _mm512_storeu_pd(output, dequantized); + _mm512_storeu_pd(output, dequantized); - output += 8; - input += BIT_WIDTH; - } + output += 8; + input += BIT_WIDTH; + } - if constexpr (remaining_elements > 0) { - const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24(source); + if constexpr (remaining_elements > 0) { + const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES + > + (source); - const __m512i converted = _mm512_cvtepi32_epi64(unpacked); + const __m512i converted = _mm512_cvtepi32_epi64(unpacked); - const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); - mm512_storeu_elements_pd(output, remaining_elements, dequantized); - } + mm512_storeu_elements_pd(output, remaining_elements, dequantized); + } - return 0; -} -} // namespace internal + return 0; + } + } // namespace internal -/** + /** * @brief Decompress a single 512\-bit block using AVX-512 and AVX-512-VBMI instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -511,24 +559,27 @@ __always_inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __re * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const float_t scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::mm512_decompress_block_avx512vbmi_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::mm512_decompress_block_avx512vbmi_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::mm512_decompress_block_avx512vbmi_17to24(input, scale, output); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::mm512_decompress_block_avx512vbmi_1to8( + input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::mm512_decompress_block_avx512vbmi_9to16( + input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::mm512_decompress_block_avx512vbmi_17to24( + input, scale, output); + } + return 0; } - return 0; -} -/** + /** * @brief Decompress a single block to double values using AVX-512 and AVX-512-VBMI instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -538,24 +589,27 @@ __always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ i * @param output pointer to the output buffer where decompressed double values will be stored. * @return int status code. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const double_t scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::mm512_decompress_block_avx512vbmi_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::mm512_decompress_block_avx512vbmi_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::mm512_decompress_block_avx512vbmi_17to24(input, scale, output); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::mm512_decompress_block_avx512vbmi_1to8( + input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::mm512_decompress_block_avx512vbmi_9to16( + input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::mm512_decompress_block_avx512vbmi_17to24( + input, scale, output); + } + return 0; } - return 0; -} -/** + /** * @brief Decompress multiple 512\-bit blocks using AVX-512 and AVX-512-VBMI instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -569,26 +623,27 @@ __always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ i * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm512_decompress_blocks_avx512vbmi(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const u8 *block_input = input; + f32 *block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + mm512_decompress_block_avx512vbmi(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } - return 0; -} + return 0; + } -/** + /** * @brief Decompress multiple blocks to double values using AVX-512 and AVX-512-VBMI instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -599,23 +654,24 @@ int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const * @param blocks number of blocks to decompress. * @return int status code. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const uint8_t* block_input = input; - double_t* block_output = output; - - for (uint32_t block = 0; block < blocks; ++block) { - mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm512_decompress_blocks_avx512vbmi(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const u8 *block_input = input; + f64 *block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + mm512_decompress_block_avx512vbmi(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + return 0; } - return 0; -} } // namespace pernix #endif // PERNIX_AVX512VBMI_DECOMPRESSION_H diff --git a/src/internal/pernix/x86/avx512vbmi/compat.h b/src/internal/pernix/x86/avx512vbmi/compat.h index 6918d66..df9ca4b 100644 --- a/src/internal/pernix/x86/avx512vbmi/compat.h +++ b/src/internal/pernix/x86/avx512vbmi/compat.h @@ -7,377 +7,377 @@ #include namespace pernix::internal { - static __always_inline __mmask8 element_mask8(const uint32_t e) { + static __always_inline __mmask8 element_mask8(const u32 e) { return static_cast<__mmask8>(e >= 8 ? 0xFFu : ((1u << e) - 1u)); } - static __always_inline __mmask16 element_mask16(const uint32_t e) { + static __always_inline __mmask16 element_mask16(const u32 e) { return static_cast<__mmask16>(e >= 16 ? 0xFFFFu : ((1u << e) - 1u)); } - static __always_inline __mmask32 element_mask32(const uint32_t e) { + static __always_inline __mmask32 element_mask32(const u32 e) { return e >= 32 ? 0xFFFFFFFFu : (1u << e) - 1u; } - static __always_inline __mmask64 element_mask64(const uint32_t e) { + static __always_inline __mmask64 element_mask64(const u32 e) { return e >= 64 ? 0xFFFFFFFFFFFFFFFFull : (1ull << e) - 1ull; } - static __always_inline __m512i mm512_loadu_elements_epi64(const uint32_t e, const void *mem_addr) { + static __always_inline __m512i mm512_loadu_elements_epi64(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int64_t)); + std::memcpy(&a, mem_addr, e * sizeof(i64)); return a; #else return _mm512_maskz_loadu_epi64(element_mask8(e), mem_addr); #endif } - static __always_inline __m256i mm256_loadu_elements_epi64(const uint32_t e, const void *mem_addr) { + static __always_inline __m256i mm256_loadu_elements_epi64(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int64_t)); + std::memcpy(&a, mem_addr, e * sizeof(i64)); return a; #else return _mm256_maskz_loadu_epi64(element_mask8(e), mem_addr); #endif } - static __always_inline __m128i mm_loadu_elements_epi64(const uint32_t e, const void *mem_addr) { + static __always_inline __m128i mm_loadu_elements_epi64(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int64_t)); + std::memcpy(&a, mem_addr, e * sizeof(i64)); return a; #else return _mm_maskz_loadu_epi64(element_mask8(e), mem_addr); #endif } - static __always_inline __m512i mm512_loadu_elements_epi32(const uint32_t e, const void *mem_addr) { + static __always_inline __m512i mm512_loadu_elements_epi32(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int32_t)); + std::memcpy(&a, mem_addr, e * sizeof(i32)); return a; #else return _mm512_maskz_loadu_epi32(element_mask16(e), mem_addr); #endif } - static __always_inline __m256i mm256_loadu_elements_epi32(const uint32_t e, const void *mem_addr) { + static __always_inline __m256i mm256_loadu_elements_epi32(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int32_t)); + std::memcpy(&a, mem_addr, e * sizeof(i32)); return a; #else return _mm256_maskz_loadu_epi32(element_mask8(e), mem_addr); #endif } - static __always_inline __m128i mm_loadu_elements_epi32(const uint32_t e, const void *mem_addr) { + static __always_inline __m128i mm_loadu_elements_epi32(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int32_t)); + std::memcpy(&a, mem_addr, e * sizeof(i32)); return a; #else return _mm_maskz_loadu_epi32(element_mask8(e), mem_addr); #endif } - static __always_inline __m512i mm512_loadu_elements_epi16(const uint32_t e, const void *mem_addr) { + static __always_inline __m512i mm512_loadu_elements_epi16(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int16_t)); + std::memcpy(&a, mem_addr, e * sizeof(i16)); return a; #else return _mm512_maskz_loadu_epi16(element_mask32(e), mem_addr); #endif } - static __always_inline __m256i mm256_loadu_elements_epi16(const uint32_t e, const void *mem_addr) { + static __always_inline __m256i mm256_loadu_elements_epi16(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int16_t)); + std::memcpy(&a, mem_addr, e * sizeof(i16)); return a; #else return _mm256_maskz_loadu_epi16(element_mask16(e), mem_addr); #endif } - static __always_inline __m128i mm_loadu_elements_epi16(const uint32_t e, const void *mem_addr) { + static __always_inline __m128i mm_loadu_elements_epi16(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int16_t)); + std::memcpy(&a, mem_addr, e * sizeof(i16)); return a; #else return _mm_maskz_loadu_epi16(element_mask8(e), mem_addr); #endif } - static __always_inline __m512i mm512_loadu_elements_epi8(const uint32_t e, const void *mem_addr) { + static __always_inline __m512i mm512_loadu_elements_epi8(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int8_t)); + std::memcpy(&a, mem_addr, e * sizeof(i8)); return a; #else return _mm512_maskz_loadu_epi8(element_mask64(e), mem_addr); #endif } - static __always_inline __m256i mm256_loadu_elements_epi8(const uint32_t e, const void *mem_addr) { + static __always_inline __m256i mm256_loadu_elements_epi8(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int8_t)); + std::memcpy(&a, mem_addr, e * sizeof(i8)); return a; #else return _mm256_maskz_loadu_epi8(element_mask32(e), mem_addr); #endif } - static __always_inline __m128i mm_loadu_elements_epi8(const uint32_t e, const void *mem_addr) { + static __always_inline __m128i mm_loadu_elements_epi8(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int8_t)); + std::memcpy(&a, mem_addr, e * sizeof(i8)); return a; #else return _mm_maskz_loadu_epi8(element_mask16(e), mem_addr); #endif } - static __always_inline void mm512_storeu_elements_epi64(void *mem_addr, const uint32_t e, const __m512i a) { + static __always_inline void mm512_storeu_elements_epi64(void *mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; + alignas(64) u8 bytes[64]; _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i64)); #else _mm512_mask_storeu_epi64(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm256_storeu_elements_epi64(void *mem_addr, const uint32_t e, const __m256i a) { + static __always_inline void mm256_storeu_elements_epi64(void *mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; + alignas(32) u8 bytes[32]; _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i64)); #else _mm256_mask_storeu_epi64(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm_storeu_elements_epi64(void *mem_addr, const uint32_t e, const __m128i a) { + static __always_inline void mm_storeu_elements_epi64(void *mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; + alignas(16) u8 bytes[16]; _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i64)); #else _mm_mask_storeu_epi64(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm512_storeu_elements_epi32(void *mem_addr, const uint32_t e, const __m512i a) { + static __always_inline void mm512_storeu_elements_epi32(void *mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; + alignas(64) u8 bytes[64]; _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i32)); #else _mm512_mask_storeu_epi32(mem_addr, element_mask16(e), a); #endif } - static __always_inline void mm256_storeu_elements_epi32(void *mem_addr, const uint32_t e, const __m256i a) { + static __always_inline void mm256_storeu_elements_epi32(void *mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; + alignas(32) u8 bytes[32]; _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i32)); #else _mm256_mask_storeu_epi32(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm_storeu_elements_epi32(void *mem_addr, const uint32_t e, const __m128i a) { + static __always_inline void mm_storeu_elements_epi32(void *mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; + alignas(16) u8 bytes[16]; _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i32)); #else _mm_mask_storeu_epi32(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm512_storeu_elements_epi16(void *mem_addr, const uint32_t e, const __m512i a) { + static __always_inline void mm512_storeu_elements_epi16(void *mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; + alignas(64) u8 bytes[64]; _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i16)); #else _mm512_mask_storeu_epi16(mem_addr, element_mask32(e), a); #endif } - static __always_inline void mm256_storeu_elements_epi16(void *mem_addr, const uint32_t e, const __m256i a) { + static __always_inline void mm256_storeu_elements_epi16(void *mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; + alignas(32) u8 bytes[32]; _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i16)); #else _mm256_mask_storeu_epi16(mem_addr, element_mask16(e), a); #endif } - static __always_inline void mm_storeu_elements_epi16(void *mem_addr, const uint32_t e, const __m128i a) { + static __always_inline void mm_storeu_elements_epi16(void *mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; + alignas(16) u8 bytes[16]; _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i16)); #else _mm_mask_storeu_epi16(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm512_storeu_elements_epi8(void *mem_addr, const uint32_t e, const __m512i a) { + static __always_inline void mm512_storeu_elements_epi8(void *mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; + alignas(64) u8 bytes[64]; _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i8)); #else _mm512_mask_storeu_epi8(mem_addr, element_mask64(e), a); #endif } - static __always_inline void mm256_storeu_elements_epi8(void *mem_addr, const uint32_t e, const __m256i a) { + static __always_inline void mm256_storeu_elements_epi8(void *mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; + alignas(32) u8 bytes[32]; _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i8)); #else _mm256_mask_storeu_epi8(mem_addr, element_mask32(e), a); #endif } - static __always_inline void mm_storeu_elements_epi8(void *mem_addr, const uint32_t e, const __m128i a) { + static __always_inline void mm_storeu_elements_epi8(void *mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; + alignas(16) u8 bytes[16]; _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i8)); #else _mm_mask_storeu_epi8(mem_addr, element_mask16(e), a); #endif } - static __always_inline __m512 mm512_loadu_elements_ps(const uint32_t e, const void *mem_addr) { + static __always_inline __m512 mm512_loadu_elements_ps(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512 a = _mm512_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(float_t)); + std::memcpy(&a, mem_addr, e * sizeof(f32)); return a; #else return _mm512_maskz_loadu_ps(element_mask16(e), mem_addr); #endif } - static __always_inline __m256 mm256_loadu_elements_ps(const uint32_t e, const void *mem_addr) { + static __always_inline __m256 mm256_loadu_elements_ps(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256 a = _mm256_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(float_t)); + std::memcpy(&a, mem_addr, e * sizeof(f32)); return a; #else return _mm256_maskz_loadu_ps(element_mask8(e), mem_addr); #endif } - static __always_inline __m128 mm_loadu_elements_ps(const uint32_t e, const void *mem_addr) { + static __always_inline __m128 mm_loadu_elements_ps(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128 a = _mm_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(float_t)); + std::memcpy(&a, mem_addr, e * sizeof(f32)); return a; #else return _mm_maskz_loadu_ps(element_mask8(e), mem_addr); #endif } - static __always_inline __m512d mm512_loadu_elements_pd(const uint32_t e, const void *mem_addr) { + static __always_inline __m512d mm512_loadu_elements_pd(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512d a = _mm512_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(double_t)); + std::memcpy(&a, mem_addr, e * sizeof(f64)); return a; #else return _mm512_maskz_loadu_pd(element_mask8(e), mem_addr); #endif } - static __always_inline __m256d mm256_loadu_elements_pd(const uint32_t e, const void *mem_addr) { + static __always_inline __m256d mm256_loadu_elements_pd(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256d a = _mm256_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(double_t)); + std::memcpy(&a, mem_addr, e * sizeof(f64)); return a; #else return _mm256_maskz_loadu_pd(element_mask8(e), mem_addr); #endif } - static __always_inline __m128d mm_loadu_elements_pd(const uint32_t e, const void *mem_addr) { + static __always_inline __m128d mm_loadu_elements_pd(const u32 e, const void *mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128d a = _mm_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(double_t)); + std::memcpy(&a, mem_addr, e * sizeof(f64)); return a; #else return _mm_maskz_loadu_pd(element_mask8(e), mem_addr); #endif } - static __always_inline void mm512_storeu_elements_ps(void *mem_addr, const uint32_t e, const __m512 a) { + static __always_inline void mm512_storeu_elements_ps(void *mem_addr, const u32 e, const __m512 a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) float_t values[16]; + alignas(64) f32 values[16]; _mm512_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(float_t)); + std::memcpy(mem_addr, values, e * sizeof(f32)); #else _mm512_mask_storeu_ps(mem_addr, element_mask16(e), a); #endif } - static __always_inline void mm256_storeu_elements_ps(void *mem_addr, const uint32_t e, const __m256 a) { + static __always_inline void mm256_storeu_elements_ps(void *mem_addr, const u32 e, const __m256 a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) float_t values[8]; + alignas(32) f32 values[8]; _mm256_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(float_t)); + std::memcpy(mem_addr, values, e * sizeof(f32)); #else _mm256_mask_storeu_ps(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm_storeu_elements_ps(void *mem_addr, const uint32_t e, const __m128 a) { + static __always_inline void mm_storeu_elements_ps(void *mem_addr, const u32 e, const __m128 a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) float_t values[4]; + alignas(16) f32 values[4]; _mm_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(float_t)); + std::memcpy(mem_addr, values, e * sizeof(f32)); #else _mm_mask_storeu_ps(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm512_storeu_elements_pd(void *mem_addr, const uint32_t e, const __m512d a) { + static __always_inline void mm512_storeu_elements_pd(void *mem_addr, const u32 e, const __m512d a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) double_t values[8]; + alignas(64) f64 values[8]; _mm512_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(double_t)); + std::memcpy(mem_addr, values, e * sizeof(f64)); #else _mm512_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm256_storeu_elements_pd(void *mem_addr, const uint32_t e, const __m256d a) { + static __always_inline void mm256_storeu_elements_pd(void *mem_addr, const u32 e, const __m256d a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) double_t values[4]; + alignas(32) f64 values[4]; _mm256_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(double_t)); + std::memcpy(mem_addr, values, e * sizeof(f64)); #else _mm256_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif } - static __always_inline void mm_storeu_elements_pd(void *mem_addr, const uint32_t e, const __m128d a) { + static __always_inline void mm_storeu_elements_pd(void *mem_addr, const u32 e, const __m128d a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) double_t values[2]; + alignas(16) f64 values[2]; _mm_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(double_t)); + std::memcpy(mem_addr, values, e * sizeof(f64)); #else _mm_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif diff --git a/src/internal/pernix/x86/avx512vbmi/packing.h b/src/internal/pernix/x86/avx512vbmi/packing.h index e51c6cc..d4052eb 100644 --- a/src/internal/pernix/x86/avx512vbmi/packing.h +++ b/src/internal/pernix/x86/avx512vbmi/packing.h @@ -9,14 +9,14 @@ namespace pernix::internal { /** * @brief Pack 8 16-bit values for bit widths 9 through 16 using VBMI. */ - template + template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i &input) { if constexpr (BIT_WIDTH == 16) { return input; } else { using tables = pack_tables_avx512_16; - const __m128i maskv = _mm_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); + const __m128i maskv = _mm_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); const __m128i masked = _mm_and_si128(input, maskv); if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { @@ -46,17 +46,17 @@ __always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i &input) { /** * @brief Pack 16 8-bit values for bit widths 1 through 8 using VBMI. */ - template + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i &input) { if constexpr (BIT_WIDTH == 8) { return input; } else { - const __m128i maskv = _mm_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); + const __m128i maskv = _mm_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); const __m128i masked = _mm_and_si128(input, maskv); if constexpr (BIT_WIDTH == 1) { - return _mm_set1_epi16(static_cast(_mm_cmpgt_epi8_mask(masked, _mm_setzero_si128()))); + return _mm_set1_epi16(static_cast(_mm_cmpgt_epi8_mask(masked, _mm_setzero_si128()))); } else if constexpr (BIT_WIDTH == 2) { const __m128i shifted = _mm_srli_epi16(masked, 6); const __m128i combined = _mm_or_si128(masked, shifted); @@ -91,12 +91,12 @@ __always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i &input) { /** * @brief Pack 4 32-bit values for bit widths 17 through 24 using VBMI. */ - template + template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i &input) { using tables = pack_tables_avx512_24; - const __m128i maskv = _mm_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); + const __m128i maskv = _mm_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); const __m128i masked = _mm_and_si128(input, maskv); const __m128 permuted1 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute1()); @@ -115,14 +115,14 @@ __always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i &input) { /** * @brief Pack 16 16-bit values for bit widths 9 through 16 using VBMI. */ - template + template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i &input) { if constexpr (BIT_WIDTH == 16) { return input; } else { using tables = pack_tables_avx512_16; - const __m256i maskv = _mm256_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); + const __m256i maskv = _mm256_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); const __m256i masked = _mm256_and_si256(input, maskv); if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { @@ -152,18 +152,18 @@ __always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i &input) /** * @brief Pack 32 8-bit values for bit widths 1 through 8 using VBMI. */ - template + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i &input) { if constexpr (BIT_WIDTH == 8) { return input; } else { - const __m256i maskv = _mm256_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); + const __m256i maskv = _mm256_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); const __m256i masked = _mm256_and_si256(input, maskv); if constexpr (BIT_WIDTH == 1) { return _mm256_set1_epi32( - static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); + static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); } else if constexpr (BIT_WIDTH == 2) { const __m256i shifted = _mm256_srli_epi16(masked, 6); const __m256i combined = _mm256_or_si256(masked, shifted); @@ -199,12 +199,12 @@ __always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i &input) { /** * @brief Pack 8 32-bit values for bit widths 17 through 24 using VBMI. */ - template + template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i &input) { using tables = pack_tables_avx512_24; - const __m256i maskv = _mm256_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); + const __m256i maskv = _mm256_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); const __m256i masked = _mm256_and_si256(input, maskv); const __m256i permuted1 = _mm256_permutexvar_epi32(tables::get_permute1(), masked); @@ -223,14 +223,14 @@ __always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i &input) /** * @brief Pack 32 16-bit values for bit widths 9 through 16 using VBMI. */ - template + template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i &input) { if constexpr (BIT_WIDTH == 16) { return input; } else { using tables = pack_tables_avx512_16; - const __m512i maskv = _mm512_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); + const __m512i maskv = _mm512_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); const __m512i masked = _mm512_and_si512(input, maskv); if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { @@ -260,18 +260,18 @@ __always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i &input) /** * @brief Pack 64 8-bit values for bit widths 1 through 8 using VBMI. */ - template + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i &input) { if constexpr (BIT_WIDTH == 8) { return input; } else { - const __m512i maskv = _mm512_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); + const __m512i maskv = _mm512_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); const __m512i masked = _mm512_and_si512(input, maskv); if constexpr (BIT_WIDTH == 1) { return _mm512_set1_epi64( - static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); + static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); } else if constexpr (BIT_WIDTH == 2) { const __m512i shifted = _mm512_srli_epi16(masked, 6); const __m512i combined = _mm512_or_si512(masked, shifted); @@ -307,12 +307,12 @@ __always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i &input) { /** * @brief Pack 16 32-bit values for bit widths 17 through 24 using VBMI. */ - template + template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i &input) { using tables = pack_tables_avx512_24; - const __m512i maskv = _mm512_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); + const __m512i maskv = _mm512_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); const __m512i masked = _mm512_and_si512(input, maskv); const __m512i permuted1 = _mm512_permutexvar_epi32(tables::get_permute1(), masked); diff --git a/src/internal/pernix/x86/avx512vbmi/tables.h b/src/internal/pernix/x86/avx512vbmi/tables.h index ab3c665..d63a47a 100644 --- a/src/internal/pernix/x86/avx512vbmi/tables.h +++ b/src/internal/pernix/x86/avx512vbmi/tables.h @@ -9,25 +9,25 @@ #include namespace pernix::internal { - template - static __always_inline Vec load_table(const std::array &table) { - static_assert(sizeof(table) >= sizeof(Vec), "table is smaller than requested SIMD vector"); - if constexpr (std::is_same_v) { - return _mm512_load_si512(static_cast(table.data())); - } else if constexpr (std::is_same_v) { - return _mm256_load_si256(reinterpret_cast(table.data())); - } else { - return _mm_load_si128(reinterpret_cast(table.data())); - } +template +static __always_inline Vec load_table(const std::array& table) { + static_assert(sizeof(table) >= sizeof(Vec), "table is smaller than requested SIMD vector"); + if constexpr (std::is_same_v) { + return _mm512_load_si512(static_cast(table.data())); + } else if constexpr (std::is_same_v) { + return _mm256_load_si256(reinterpret_cast(table.data())); + } else { + return _mm_load_si128(reinterpret_cast(table.data())); } +} - template - requires(N >= 9 && N <= 15) - struct pack_tables_avx512_16 { - alignas(64) inline static constexpr std::array permute1 = [] { +template + requires(N >= 9 && N <= 15) +struct pack_tables_avx512_16 { + alignas(64) inline static constexpr std::array permute1 = [] { // clang-format off if constexpr (N == 9) { - return std::array{ + return std::array{ 0, 2, 4, 6, -1, 9, 11, 13, 15, 16, 18, 20, @@ -39,7 +39,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 10) { - return std::array{ + return std::array{ 0, 2, -1, 5, 7, 8, 10, -1, 13, 15, 16, 18, @@ -51,7 +51,7 @@ namespace pernix::internal { 45, 47, -1, -1 }; } else if constexpr (N == 11) { - return std::array{ + return std::array{ 0, 2, 3, 5, 6, -1, 9, -1, 12, -1, 15, 16, @@ -63,7 +63,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 12) { - return std::array{ + return std::array{ 1, 2, 3, 5, 6, 7, 9, 10, 11, 13, 14, 15, @@ -75,7 +75,7 @@ namespace pernix::internal { 38, 39, 41, 42 }; } else if constexpr (N == 13) { - return std::array{ + return std::array{ 0, -1, 3, 4, 5, -1, -1, 9, 10, -1, -1, 14, @@ -87,7 +87,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 14) { - return std::array{ + return std::array{ 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, @@ -99,7 +99,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 15) { - return std::array{ + return std::array{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, @@ -111,14 +111,14 @@ namespace pernix::internal { 30, 31, -1, -1 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute2 = [] { + alignas(64) inline static constexpr std::array permute2 = [] { // clang-format off if constexpr (N == 9) { - return std::array{ + return std::array{ 1, 3, 5, 7, 8, 10, 12, 14, -1, 17, 19, 21, @@ -130,7 +130,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 10) { - return std::array{ + return std::array{ 1, 3, 4, 6, -1, 9, 11, 12, 14, -1, 17, 19, @@ -142,7 +142,7 @@ namespace pernix::internal { 46, -1, -1, -1 }; } else if constexpr (N == 11) { - return std::array{ + return std::array{ 1, -1, 4, -1, 7, 8, 10, 11, 13, 14, -1, 17, @@ -154,7 +154,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 12) { - return std::array{ + return std::array{ 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, @@ -166,7 +166,7 @@ namespace pernix::internal { 37, 38, 40, 41 }; } else if constexpr (N == 13) { - return std::array{ + return std::array{ 1, 2, -1, -1, 6, 7, 8, -1, 11, 12, 13, -1, @@ -178,7 +178,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 14) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, @@ -190,7 +190,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 15) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, @@ -202,13 +202,13 @@ namespace pernix::internal { 29, 30, -1, -1 }; } - // clang-format on - }(); + // clang-format on + }(); - alignas(64) inline static constexpr std::array permute3 = [] { + alignas(64) inline static constexpr std::array permute3 = [] { // clang-format off if constexpr (N == 9) { - return std::array{ + return std::array{ -1, 1, 3, 5, 7, 8, 10, 12, 14, -1, 17, 19, @@ -220,7 +220,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 10) { - return std::array{ + return std::array{ -1, 1, 3, 4, 6, -1, 9, 11, 12, 14, -1, 17, @@ -232,7 +232,7 @@ namespace pernix::internal { 44, 46, -1, -1 }; } else if constexpr (N == 11) { - return std::array{ + return std::array{ -1, 1, 2, 4, 5, 7, 8, 10, 11, 13, 14, -1, @@ -244,7 +244,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 13) { - return std::array{ + return std::array{ -1, 1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13, @@ -256,14 +256,14 @@ namespace pernix::internal { -1, -1, -1, -1 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift1 = [] { + alignas(64) inline static constexpr std::array shift1 = [] { // clang-format off if constexpr (N == 9) { - return std::array{ + return std::array{ 0, 2, 4, 6, 0, 1, 3, 5, 7, 0, 2, 4, @@ -275,7 +275,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 10) { - return std::array{ + return std::array{ 0, 4, 0, 2, 6, 0, 4, 0, 2, 6, 0, 4, @@ -287,7 +287,7 @@ namespace pernix::internal { 2, 6, -1, -1 }; } else if constexpr (N == 11) { - return std::array{ + return std::array{ 0, 6, 1, 7, 2, 0, 3, 0, 4, 0, 5, 0, @@ -299,7 +299,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 12) { - return std::array{ + return std::array{ 12, 8, 4, 12, 8, 4, 12, 8, 4, 12, 8, 4, @@ -311,7 +311,7 @@ namespace pernix::internal { 8, 4, 12, 8 }; } else if constexpr (N == 13) { - return std::array{ + return std::array{ 0, 0, 7, 4, 1, 0, 0, 5, 2, 0, 0, 6, @@ -323,7 +323,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 14) { - return std::array{ + return std::array{ 14, 12, 10, 8, 6, 4, 2, 14, 12, 10, 8, 6, @@ -335,7 +335,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 15) { - return std::array{ + return std::array{ 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, @@ -347,14 +347,14 @@ namespace pernix::internal { 2, 1, -1, -1 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift2 = [] { + alignas(64) inline static constexpr std::array shift2 = [] { // clang-format off if constexpr (N == 9) { - return std::array{ + return std::array{ 9, 11, 13, 15, 8, 10, 12, 14, 8, 9, 11, 13, @@ -366,7 +366,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 10) { - return std::array{ + return std::array{ 10, 14, 8, 12, 0, 10, 14, 8, 12, 0, 10, 14, @@ -378,7 +378,7 @@ namespace pernix::internal { 12, 0, -1, -1 }; } else if constexpr (N == 11) { - return std::array{ + return std::array{ 11, 8, 12, 8, 13, 8, 14, 9, 15, 10, 8, 11, @@ -390,7 +390,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 12) { - return std::array{ + return std::array{ 0, 4, 8, 0, 4, 8, 0, 4, 8, 0, 4, 8, @@ -402,7 +402,7 @@ namespace pernix::internal { 4, 8, 0, 4 }; } else if constexpr (N == 13) { - return std::array{ + return std::array{ 13, 10, 0, 0, 14, 11, 8, 0, 15, 12, 9, 0, @@ -414,7 +414,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 14) { - return std::array{ + return std::array{ 0, 2, 4, 6, 8, 10, 12, 0, 2, 4, 6, 8, @@ -426,7 +426,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 15) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, @@ -438,14 +438,14 @@ namespace pernix::internal { 13, 14, -1, -1 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - alignas(64) inline static constexpr std::array shift3 = [] { + alignas(64) inline static constexpr std::array shift3 = [] { // clang-format off if constexpr (N == 9) { - return std::array{ + return std::array{ 0, 7, 5, 3, 1, 8, 6, 4, 2, 0, 7, 5, @@ -457,7 +457,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 10) { - return std::array{ + return std::array{ 0, 6, 2, 8, 4, 0, 6, 2, 8, 4, 0, 6, @@ -469,7 +469,7 @@ namespace pernix::internal { 8, 4, -1, -1 }; } else if constexpr (N == 11) { - return std::array{ + return std::array{ 0, 5, 10, 4, 9, 3, 8, 2, 7, 1, 6, 0, @@ -481,7 +481,7 @@ namespace pernix::internal { -1, -1, -1, -1 }; } else if constexpr (N == 13) { - return std::array{ + return std::array{ 0, 3, 6, 9, 12, 2, 5, 8, 11, 1, 4, 7, @@ -493,11 +493,11 @@ namespace pernix::internal { -1, -1, -1, -1 }; } - return std::array{}; - // clang-format on - }(); + return std::array{}; + // clang-format on + }(); - inline static constexpr std::tuple<__mmask32, __mmask32, __mmask32> get_permute_masks() { + inline static constexpr std::tuple<__mmask32, __mmask32, __mmask32> get_permute_masks() { // clang-format off if constexpr (N == 9) { return { @@ -525,305 +525,305 @@ namespace pernix::internal { }; } return {0, 0, 0}; - // clang-format on - } - - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } - static __always_inline Vec get_permute3() { return load_table(permute3); } + // clang-format on + } - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } - static __always_inline Vec get_shift3() { return load_table(shift3); } + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } + static __always_inline Vec get_permute3() { return load_table(permute3); } + + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } + static __always_inline Vec get_shift3() { return load_table(shift3); } +}; + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && + (std::is_same_v || std::is_same_v || std::is_same_v)) +struct pack_tables_avx512_24 { +private: + struct word_plan { + i32 left_index1 = -1; + i32 left_index2 = -1; + i32 right_index = -1; + u32 left_shift1 = 32; + u32 left_shift2 = 32; + u32 right_shift = 32; }; - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && - (std::is_same_v || std::is_same_v || std::is_same_v)) - struct pack_tables_avx512_24 { - private: - struct word_plan { - int32_t left_index1 = -1; - int32_t left_index2 = -1; - int32_t right_index = -1; - uint32_t left_shift1 = 32; - uint32_t left_shift2 = 32; - uint32_t right_shift = 32; - }; - - static constexpr word_plan create_plan(const uint32_t idx) { - word_plan plan{}; - - const uint32_t word_start = idx * 32u; - const uint32_t word_end = word_start + 32u; - - uint32_t left_slot = 0; - for (uint32_t input_lane = 0; input_lane < 16; ++input_lane) { - const uint32_t input_start = input_lane * BIT_WIDTH; - const uint32_t input_end = input_start + BIT_WIDTH; - - const uint32_t overlap_start = std::max(word_start, input_start); - const uint32_t overlap_end = std::min(word_end, input_end); - if (overlap_start >= overlap_end) { - continue; - } + static constexpr word_plan create_plan(const u32 idx) { + word_plan plan{}; - const auto output_bit = static_cast(overlap_start - word_start); - const auto input_bit = static_cast(overlap_start - input_start); - const int32_t delta = output_bit - input_bit; - - if (delta >= 0) { - if (left_slot == 0) { - plan.left_index1 = static_cast(input_lane); - plan.left_shift1 = static_cast(delta); - ++left_slot; - } else { - plan.left_index2 = static_cast(input_lane); - plan.left_shift2 = static_cast(delta); - } - } else { - plan.right_index = static_cast(input_lane); - plan.right_shift = static_cast(-delta); - } - } + const u32 word_start = idx * 32u; + const u32 word_end = word_start + 32u; - return plan; - } + u32 left_slot = 0; + for (u32 input_lane = 0; input_lane < 16; ++input_lane) { + const u32 input_start = input_lane * BIT_WIDTH; + const u32 input_end = input_start + BIT_WIDTH; - static constexpr std::array word_plans = [] { - std::array plans{}; - for (uint32_t i = 0; i < 16; ++i) { - plans[i] = create_plan(i); - } - return plans; - }(); - - template - static __always_inline constexpr std::array make_table(Getter getter) { - std::array values{}; - for (uint32_t i = 0; i < 16; ++i) { - values[i] = getter(word_plans[i]); + const u32 overlap_start = std::max(word_start, input_start); + const u32 overlap_end = std::min(word_end, input_end); + if (overlap_start >= overlap_end) { + continue; } - return values; - } - - alignas(64) static constexpr auto permute1 = make_table([](const word_plan &p) { - return p.left_index1; - }); - - alignas(64) static constexpr auto permute2 = make_table([](const word_plan &p) { - return p.left_index2; - }); - alignas(64) static constexpr auto permute3 = make_table([](const word_plan &p) { - return p.right_index; - }); + const auto output_bit = static_cast(overlap_start - word_start); + const auto input_bit = static_cast(overlap_start - input_start); + const i32 delta = output_bit - input_bit; - alignas(64) static constexpr auto shift1 = make_table( - [](const word_plan &p) { return p.left_shift1; }); - - alignas(64) static constexpr auto shift2 = make_table( - [](const word_plan &p) { return p.left_shift2; }); - - alignas(64) static constexpr auto shift3 = make_table( - [](const word_plan &p) { return p.right_shift; }); - - public: - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } - static __always_inline Vec get_permute3() { return load_table(permute3); } - - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } - static __always_inline Vec get_shift3() { return load_table(shift3); } - }; - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && - (std::is_same_v || std::is_same_v || std::is_same_v)) - struct unpack_tables_avx512_8 { - private: - alignas(64) inline static constexpr std::array permute1 = [] { - std::array table{}; - std::ranges::fill(table, -1); - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - - table[entry] = static_cast(first_byte); + if (delta >= 0) { + if (left_slot == 0) { + plan.left_index1 = static_cast(input_lane); + plan.left_shift1 = static_cast(delta); + ++left_slot; + } else { + plan.left_index2 = static_cast(input_lane); + plan.left_shift2 = static_cast(delta); + } + } else { + plan.right_index = static_cast(input_lane); + plan.right_shift = static_cast(-delta); } + } - return table; - }(); - - alignas(64) inline static constexpr std::array permute2 = [] { - std::array table{}; - std::ranges::fill(table, -1); + return plan; + } - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t bit_offset = bit_start % 8; + static constexpr std::array word_plans = [] { + std::array plans{}; + for (u32 i = 0; i < 16; ++i) { + plans[i] = create_plan(i); + } + return plans; + }(); + + template + static __always_inline constexpr std::array make_table(Getter getter) { + std::array values{}; + for (u32 i = 0; i < 16; ++i) { + values[i] = getter(word_plans[i]); + } + return values; + } - if (bit_offset + BIT_WIDTH > 8) { - table[entry] = static_cast(first_byte + 1); - } - } + alignas(64) static constexpr auto permute1 = make_table([](const word_plan& p) { + return p.left_index1; + }); + + alignas(64) static constexpr auto permute2 = make_table([](const word_plan& p) { + return p.left_index2; + }); + + alignas(64) static constexpr auto permute3 = make_table([](const word_plan& p) { + return p.right_index; + }); + + alignas(64) static constexpr auto shift1 = make_table( + [](const word_plan& p) { return p.left_shift1; }); + + alignas(64) static constexpr auto shift2 = make_table( + [](const word_plan& p) { return p.left_shift2; }); + + alignas(64) static constexpr auto shift3 = make_table( + [](const word_plan& p) { return p.right_shift; }); + +public: + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } + static __always_inline Vec get_permute3() { return load_table(permute3); } + + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } + static __always_inline Vec get_shift3() { return load_table(shift3); } +}; + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && + (std::is_same_v || std::is_same_v || std::is_same_v)) +struct unpack_tables_avx512_8 { +private: + alignas(64) inline static constexpr std::array permute1 = [] { + std::array table{}; + std::ranges::fill(table, -1); + for (size_t entry = 0; entry < 64; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t first_byte = bit_start / 8; + + table[entry] = static_cast(first_byte); + } - return table; - }(); + return table; + }(); - alignas(64) inline static constexpr std::array shift1 = [] { - std::array table{}; + alignas(64) inline static constexpr std::array permute2 = [] { + std::array table{}; + std::ranges::fill(table, -1); - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8; + for (size_t entry = 0; entry < 64; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t first_byte = bit_start / 8; + const size_t bit_offset = bit_start % 8; - table[entry] = static_cast(bit_offset); + if (bit_offset + BIT_WIDTH > 8) { + table[entry] = static_cast(first_byte + 1); } + } - return table; - }(); + return table; + }(); - alignas(64) inline static constexpr std::array shift2 = [] { - std::array table{}; + alignas(64) inline static constexpr std::array shift1 = [] { + std::array table{}; - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - const size_t spill_bits = (bit_offset + BIT_WIDTH > 8u) ? (bit_offset + BIT_WIDTH - 8u) : 0u; + for (size_t entry = 0; entry < 64; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_offset = bit_start % 8; - table[entry] = spill_bits ? static_cast(8 - bit_offset) : 0; - } + table[entry] = static_cast(bit_offset); + } - return table; - }(); + return table; + }(); - public: - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } + alignas(64) inline static constexpr std::array shift2 = [] { + std::array table{}; - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } - }; + for (size_t entry = 0; entry < 64; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_offset = bit_start % 8u; + const size_t spill_bits = (bit_offset + BIT_WIDTH > 8u) ? (bit_offset + BIT_WIDTH - 8u) : 0u; - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && - (std::is_same_v || std::is_same_v || std::is_same_v)) - struct unpack_tables_avx512_16 { - private: - alignas(64) inline static constexpr std::array permute1 = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t base = entry * 2; - - table[base] = static_cast(first_byte); - table[base + 1] = static_cast(first_byte + 1); - } + table[entry] = spill_bits ? static_cast(8 - bit_offset) : 0; + } - return table; - }(); + return table; + }(); + +public: + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } + + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } +}; + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && + (std::is_same_v || std::is_same_v || std::is_same_v)) +struct unpack_tables_avx512_16 { +private: + alignas(64) inline static constexpr std::array permute1 = [] { + std::array table{}; + std::ranges::fill(table, -1); + + for (size_t entry = 0; entry < 32; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t first_byte = bit_start / 8; + const size_t base = entry * 2; + + table[base] = static_cast(first_byte); + table[base + 1] = static_cast(first_byte + 1); + } - alignas(64) inline static constexpr std::array permute2 = [] { - std::array table{}; - std::ranges::fill(table, -1); + return table; + }(); - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t bit_offset = bit_start % 8; - const size_t base = entry * 2; + alignas(64) inline static constexpr std::array permute2 = [] { + std::array table{}; + std::ranges::fill(table, -1); - if (bit_offset + BIT_WIDTH > 16) { - table[base] = static_cast(first_byte + 2); - } + for (size_t entry = 0; entry < 32; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t first_byte = bit_start / 8; + const size_t bit_offset = bit_start % 8; + const size_t base = entry * 2; + + if (bit_offset + BIT_WIDTH > 16) { + table[base] = static_cast(first_byte + 2); } + } - return table; - }(); + return table; + }(); - alignas(64) inline static constexpr std::array shift1 = [] { - std::array table{}; + alignas(64) inline static constexpr std::array shift1 = [] { + std::array table{}; - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; + for (size_t entry = 0; entry < 32; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_offset = bit_start % 8u; - // Right-shift the 16-bit chunk so the value starts at bit 0. - table[entry] = static_cast(bit_offset); - } + // Right-shift the 16-bit chunk so the value starts at bit 0. + table[entry] = static_cast(bit_offset); + } - return table; - }(); + return table; + }(); - alignas(64) inline static constexpr std::array shift2 = [] { - std::array table{}; - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - const size_t spill_bits = (bit_offset + BIT_WIDTH > 16u) ? (bit_offset + BIT_WIDTH - 16u) : 0u; + alignas(64) inline static constexpr std::array shift2 = [] { + std::array table{}; + for (size_t entry = 0; entry < 32; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_offset = bit_start % 8u; + const size_t spill_bits = (bit_offset + BIT_WIDTH > 16u) ? (bit_offset + BIT_WIDTH - 16u) : 0u; - // Move spill bits from byte3 to their final bit positions before merge. - table[entry] = spill_bits ? static_cast(16u - bit_offset) : 0; - } + // Move spill bits from byte3 to their final bit positions before merge. + table[entry] = spill_bits ? static_cast(16u - bit_offset) : 0; + } - return table; - }(); + return table; + }(); - public: - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } +public: + static __always_inline Vec get_permute1() { return load_table(permute1); } + static __always_inline Vec get_permute2() { return load_table(permute2); } - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } - }; + static __always_inline Vec get_shift1() { return load_table(shift1); } + static __always_inline Vec get_shift2() { return load_table(shift2); } +}; - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && - (std::is_same_v || std::is_same_v || std::is_same_v)) - struct unpack_tables_avx512_24 { - private: - alignas(64) inline static constexpr std::array permute = [] { - std::array table{}; - std::ranges::fill(table, -1); +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && + (std::is_same_v || std::is_same_v || std::is_same_v)) +struct unpack_tables_avx512_24 { +private: + alignas(64) inline static constexpr std::array permute = [] { + std::array table{}; + std::ranges::fill(table, -1); - for (size_t entry = 0; entry < 16; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_end = bit_start + BIT_WIDTH - 1; + for (size_t entry = 0; entry < 16; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + const size_t bit_end = bit_start + BIT_WIDTH - 1; - const size_t first_byte = bit_start / 8; - const size_t last_byte = bit_end / 8; + const size_t first_byte = bit_start / 8; + const size_t last_byte = bit_end / 8; - const size_t base = entry * 4; + const size_t base = entry * 4; - for (size_t byte = first_byte; byte <= last_byte; ++byte) { - table[base + (byte - first_byte)] = static_cast(byte); - } + for (size_t byte = first_byte; byte <= last_byte; ++byte) { + table[base + (byte - first_byte)] = static_cast(byte); } + } - return table; - }(); + return table; + }(); - alignas(64) inline static constexpr std::array shift = [] { - std::array table{}; + alignas(64) inline static constexpr std::array shift = [] { + std::array table{}; - for (size_t entry = 0; entry < 16; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - table[entry] = static_cast(32u - BIT_WIDTH - (bit_start % 8u)); - } + for (size_t entry = 0; entry < 16; ++entry) { + const size_t bit_start = entry * BIT_WIDTH; + table[entry] = static_cast(32u - BIT_WIDTH - (bit_start % 8u)); + } - return table; - }(); + return table; + }(); - public: - static __always_inline Vec get_permute() { return load_table(permute); } - static __always_inline Vec get_shift() { return load_table(shift); } - }; +public: + static __always_inline Vec get_permute() { return load_table(permute); } + static __always_inline Vec get_shift() { return load_table(shift); } +}; } // namespace pernix::internal #endif // PERNIX_AVX512VBMI_TABLES_H diff --git a/src/internal/pernix/x86/avx512vbmi/unpacking.h b/src/internal/pernix/x86/avx512vbmi/unpacking.h index 6799cd2..e66f9ec 100644 --- a/src/internal/pernix/x86/avx512vbmi/unpacking.h +++ b/src/internal/pernix/x86/avx512vbmi/unpacking.h @@ -22,7 +22,7 @@ __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i coun return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); } -__always_inline static __m128i _mm_slli_epi8(const __m128i a, const int8_t imm8) { +__always_inline static __m128i _mm_slli_epi8(const __m128i a, const i8 imm8) { return _mm_sllv_epi8(a, _mm_set1_epi8(imm8)); } @@ -37,7 +37,7 @@ __always_inline static __m128i _mm_srli_epi8(const __m128i a, const int imm8) { return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); } -__always_inline static __m128i _mm_srai_epi8(const __m128i a, const int8_t imm8) { +__always_inline static __m128i _mm_srai_epi8(const __m128i a, const i8 imm8) { const __m128i lo_mask = _mm_set1_epi16(0x00ff); const __m128i hi_mask = _mm_set1_epi16(0xff00); const __m128i shift = _mm_cvtsi32_si128(imm8); @@ -50,7 +50,7 @@ __always_inline static __m128i _mm_srai_epi8(const __m128i a, const int8_t imm8) return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); } - template + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i &input) { if constexpr (BIT_WIDTH == 8) { @@ -107,7 +107,7 @@ __always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i &input) { const __mmask16 spill_mask = _mm_cmpneq_epi8_mask(tables::get_shift2(), _mm_setzero_si128()); __m128i combined = _mm_or_si128(shifted1, _mm_maskz_mov_epi8(spill_mask, shifted2)); - constexpr uint32_t shift = 8 - BIT_WIDTH; + constexpr u32 shift = 8 - BIT_WIDTH; combined = _mm_slli_epi8(combined, shift); if (SIGN_VALUES) { combined = _mm_srai_epi8(combined, shift); @@ -120,7 +120,7 @@ __always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i &input) { } } - template + template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i &input) { if constexpr (BIT_WIDTH == 16) { @@ -138,7 +138,7 @@ __always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i &input) { shifted = _mm_or_si128(shifted, shifted2); } - constexpr uint32_t shift = 16 - BIT_WIDTH; + constexpr u32 shift = 16 - BIT_WIDTH; shifted = _mm_slli_epi16(shifted, shift); if (SIGN_VALUES) { shifted = _mm_srai_epi16(shifted, shift); @@ -150,14 +150,14 @@ __always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i &input) { } } - template + template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i &input) { using tables = unpack_tables_avx512_24; const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute(), input); - constexpr uint32_t shift = 32 - BIT_WIDTH; + constexpr u32 shift = 32 - BIT_WIDTH; __m128i shifted = _mm_sllv_epi32(permuted, tables::get_shift()); if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { shifted = _mm_srai_epi32(shifted, shift); @@ -186,11 +186,11 @@ __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i c return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); } -__always_inline static __m256i _mm256_slli_epi8(const __m256i a, const int8_t imm8) { +__always_inline static __m256i _mm256_slli_epi8(const __m256i a, const i8 imm8) { return _mm256_sllv_epi8(a, _mm256_set1_epi8(imm8)); } -__always_inline static __m256i _mm256_srli_epi8(const __m256i a, const int8_t imm8) { +__always_inline static __m256i _mm256_srli_epi8(const __m256i a, const i8 imm8) { const __m256i lo_mask = _mm256_set1_epi16(0x00ff); const __m256i hi_mask = _mm256_set1_epi16(0xff00); const __m128i shift = _mm_cvtsi32_si128(imm8); @@ -201,7 +201,7 @@ __always_inline static __m256i _mm256_srli_epi8(const __m256i a, const int8_t im return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); } -__always_inline static __m256i _mm256_srai_epi8(const __m256i a, const int8_t imm8) { +__always_inline static __m256i _mm256_srai_epi8(const __m256i a, const i8 imm8) { const __m256i lo_mask = _mm256_set1_epi16(0x00ff); const __m256i hi_mask = _mm256_set1_epi16(0xff00); const __m128i shift = _mm_cvtsi32_si128(imm8); @@ -214,7 +214,7 @@ __always_inline static __m256i _mm256_srai_epi8(const __m256i a, const int8_t im return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); } - template + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i &input) { if constexpr (BIT_WIDTH == 8) { @@ -271,7 +271,7 @@ __always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i &input) const __mmask32 spill_mask = _mm256_cmpneq_epi8_mask(tables::get_shift2(), _mm256_setzero_si256()); __m256i combined = _mm256_or_si256(shifted1, _mm256_maskz_mov_epi8(spill_mask, shifted2)); - constexpr uint32_t shift = 8 - BIT_WIDTH; + constexpr u32 shift = 8 - BIT_WIDTH; combined = _mm256_slli_epi8(combined, shift); if (SIGN_VALUES) { combined = _mm256_srai_epi8(combined, shift); @@ -284,7 +284,7 @@ __always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i &input) } } - template + template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i &input) { if constexpr (BIT_WIDTH == 16) { @@ -302,7 +302,7 @@ __always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i &input shifted = _mm256_or_si256(shifted, shifted2); } - constexpr uint32_t shift = 16 - BIT_WIDTH; + constexpr u32 shift = 16 - BIT_WIDTH; shifted = _mm256_slli_epi16(shifted, shift); if (SIGN_VALUES) { shifted = _mm256_srai_epi16(shifted, shift); @@ -314,14 +314,14 @@ __always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i &input } } - template + template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i &input) { using tables = unpack_tables_avx512_24; const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute(), input); - constexpr uint32_t shift = 32 - BIT_WIDTH; + constexpr u32 shift = 32 - BIT_WIDTH; __m256i shifted = _mm256_sllv_epi32(permuted, tables::get_shift()); if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { shifted = _mm256_srai_epi32(shifted, shift); @@ -350,11 +350,11 @@ __always_inline static __m512i _mm512_sllv_epi8(const __m512i a, const __m512i c return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); } -__always_inline static __m512i _mm512_slli_epi8(const __m512i a, const int8_t imm8) { +__always_inline static __m512i _mm512_slli_epi8(const __m512i a, const i8 imm8) { return _mm512_sllv_epi8(a, _mm512_set1_epi8(imm8)); } -__always_inline static __m512i _mm512_srli_epi8(const __m512i a, const int8_t imm8) { +__always_inline static __m512i _mm512_srli_epi8(const __m512i a, const i8 imm8) { const __m512i lo_mask = _mm512_set1_epi16(0x00ff); const __m512i hi_mask = _mm512_set1_epi16(0xff00); const __m128i shift = _mm_cvtsi32_si128(imm8); @@ -365,7 +365,7 @@ __always_inline static __m512i _mm512_srli_epi8(const __m512i a, const int8_t im return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); } -__always_inline static __m512i _mm512_srai_epi8(const __m512i a, const int8_t imm8) { +__always_inline static __m512i _mm512_srai_epi8(const __m512i a, const i8 imm8) { const __m512i lo_mask = _mm512_set1_epi16(0x00ff); const __m512i hi_mask = _mm512_set1_epi16(0xff00); const __m128i shift = _mm_cvtsi32_si128(imm8); @@ -378,7 +378,7 @@ __always_inline static __m512i _mm512_srai_epi8(const __m512i a, const int8_t im return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); } - template + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i &input) { if constexpr (BIT_WIDTH == 8) { @@ -435,7 +435,7 @@ __always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i &input) const __mmask64 spill_mask = _mm512_cmpneq_epi8_mask(tables::get_shift2(), _mm512_setzero_si512()); __m512i combined = _mm512_or_si512(shifted1, _mm512_maskz_mov_epi8(spill_mask, shifted2)); - constexpr uint32_t shift = 8 - BIT_WIDTH; + constexpr u32 shift = 8 - BIT_WIDTH; combined = _mm512_slli_epi8(combined, shift); if (SIGN_VALUES) { combined = _mm512_srai_epi8(combined, shift); @@ -448,7 +448,7 @@ __always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i &input) } } - template + template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i &input) { if constexpr (BIT_WIDTH == 16) { @@ -465,7 +465,7 @@ __always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i &input shifted = _mm512_or_si512(shifted, shifted2); } - constexpr uint32_t shift = 16 - BIT_WIDTH; + constexpr u32 shift = 16 - BIT_WIDTH; shifted = _mm512_slli_epi16(shifted, shift); if (SIGN_VALUES) { shifted = _mm512_srai_epi16(shifted, shift); @@ -477,7 +477,7 @@ __always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i &input } } - template + template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i &input) { using tables = unpack_tables_avx512_24; @@ -485,7 +485,7 @@ __always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i &inpu const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute(), input); __m512i shifted = _mm512_sllv_epi32(permuted, tables::get_shift()); - constexpr uint32_t shift = 32 - BIT_WIDTH; + constexpr u32 shift = 32 - BIT_WIDTH; if constexpr (SIGN_VALUES) { shifted = _mm512_srai_epi32(shifted, shift); } else { diff --git a/src/internal/pernix/x86/bmi2/bmi2_compression.h b/src/internal/pernix/x86/bmi2/bmi2_compression.h index 2f2d362..e165c3a 100644 --- a/src/internal/pernix/x86/bmi2/bmi2_compression.h +++ b/src/internal/pernix/x86/bmi2/bmi2_compression.h @@ -10,147 +10,147 @@ #include namespace pernix { -namespace internal { -/** + namespace internal { + /** * @brief Build the masks and shift constants used by the BMI2 packers. * * @tparam BIT_WIDTH bit width per packed value. -* @return std::tuple mask tuple used by the BMI2 helpers. +* @return std::tuple mask tuple used by the BMI2 helpers. */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) -static constexpr std::tuple pack_avx2_bmi2_constants() { - uint32_t mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - uint64_t pext_mask; - uint16_t shift1 = BIT_WIDTH * 4; - uint16_t shift2 = 64 - shift1; - - if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { - pext_mask = 0x0101010101010101ULL * mask; - } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { - pext_mask = 0x0001000100010001ULL * mask; - } else { - pext_mask = 0x0000000100000001ULL * mask; - } + template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) + static constexpr std::tuple pack_avx2_bmi2_constants() { + u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + u64 pext_mask; + u16 shift1 = BIT_WIDTH * 4; + u16 shift2 = 64 - shift1; + + if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { + pext_mask = 0x0101010101010101ULL * mask; + } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { + pext_mask = 0x0001000100010001ULL * mask; + } else { + pext_mask = 0x0000000100000001ULL * mask; + } - return { - mask, - pext_mask, - shift1, - shift2, - }; -} + return { + mask, + pext_mask, + shift1, + shift2, + }; + } -/** + /** * @brief Pack four 32-bit values with BMI2 extract instructions. * * @tparam BIT_WIDTH bit width per packed value. * @param input SIMD register containing four quantized values. * @return __m128i packed bitstream in the low bytes of the result. */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) -static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { - const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); - - if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 16) { - const __m128i packed = _mm_packs_epi32(input, _mm_setzero_si128()); - const uint64_t value = _pext_u64(_mm_extract_epi64(packed, 0), pext_mask); - - const __m128i result = _mm_set_epi64x(0, value); - return result; - } else { - alignas(16) uint64_t values[2]; - values[0] = _pext_u64(_mm_extract_epi64(input, 0), pext_mask); - - const uint64_t temp_combined = _pext_u64(_mm_extract_epi64(input, 1), pext_mask); - values[1] = temp_combined >> shift2; - values[0] |= (temp_combined << shift1); - - const __m128i result = _mm_set_epi64x(static_cast(values[1]), static_cast(values[0])); - return result; - } -} + template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) + static inline auto mm_pack_epi32_bmi2(const __m128i &input) -> __m128i { + const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); + + if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 16) { + const __m128i packed = _mm_packs_epi32(input, _mm_setzero_si128()); + const u64 value = _pext_u64(_mm_extract_epi64(packed, 0), pext_mask); -/** + const __m128i result = _mm_set_epi64x(0, value); + return result; + } else { + alignas(16) u64 values[2]; + values[0] = _pext_u64(_mm_extract_epi64(input, 0), pext_mask); + + const u64 temp_combined = _pext_u64(_mm_extract_epi64(input, 1), pext_mask); + values[1] = temp_combined >> shift2; + values[0] |= (temp_combined << shift1); + + const __m128i result = _mm_set_epi64x(static_cast(values[1]), static_cast(values[0])); + return result; + } + } + + /** * @brief Pack eight 32-bit values with BMI2 extract instructions. * * @tparam BIT_WIDTH bit width per packed value. * @param input SIMD register containing eight quantized values. * @return __m256i packed bitstream in the low bytes of the result. */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { - const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); - - if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { - const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); - const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); - const __m256i packed8 = _mm256_packs_epi16(permuted, _mm256_setzero_si256()); - const uint64_t value = _pext_u64(_mm256_extract_epi64(packed8, 0), pext_mask); - - const __m256i result = _mm256_setr_epi64x(static_cast(value), 0, 0, 0); - return result; - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); - alignas(16) int64_t values[2] = {}; - values[0] = _pext_u64(_mm256_extract_epi64(packed16, 0), pext_mask); - - const uint64_t temp_combined = _pext_u64(_mm256_extract_epi64(packed16, 2), pext_mask); - values[1] = temp_combined >> shift2; - if constexpr (BIT_WIDTH != 16) { - values[0] |= static_cast(temp_combined << shift1); - } - - const __m256i result = _mm256_setr_epi64x(values[0], values[1], 0, 0); - return result; - } else { - constexpr uint32_t chunk_bits = BIT_WIDTH * 2; // bits extracted per 64-bit lane - static_assert(chunk_bits < 64); - - constexpr uint64_t chunk_mask = (chunk_bits == 64) ? ~uint64_t{0} : ((uint64_t{1} << chunk_bits) - 1); - - const uint64_t x0 = _pext_u64(_mm256_extract_epi64(input, 0), pext_mask) & chunk_mask; - const uint64_t x1 = _pext_u64(_mm256_extract_epi64(input, 1), pext_mask) & chunk_mask; - const uint64_t x2 = _pext_u64(_mm256_extract_epi64(input, 2), pext_mask) & chunk_mask; - const uint64_t x3 = _pext_u64(_mm256_extract_epi64(input, 3), pext_mask) & chunk_mask; - - uint64_t out0 = 0; - uint64_t out1 = 0; - uint64_t out2 = 0; - - auto append_bits = [&](uint64_t value, uint32_t bit_offset) { - const uint32_t word = bit_offset >> 6; // / 64 - const uint32_t off = bit_offset & 63; // % 64 - - if (word == 0) { - out0 |= value << off; - if (off + chunk_bits > 64) { - out1 |= value >> (64 - off); - } - } else if (word == 1) { - out1 |= value << off; - if (off + chunk_bits > 64) { - out2 |= value >> (64 - off); + template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) + static inline auto mm256_pack_epi32_bmi2(const __m256i &input) -> __m256i { + const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); + + if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { + const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); + const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); + const __m256i packed8 = _mm256_packs_epi16(permuted, _mm256_setzero_si256()); + const u64 value = _pext_u64(_mm256_extract_epi64(packed8, 0), pext_mask); + + const __m256i result = _mm256_setr_epi64x(static_cast(value), 0, 0, 0); + return result; + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); + alignas(16) i64 values[2] = {}; + values[0] = _pext_u64(_mm256_extract_epi64(packed16, 0), pext_mask); + + const u64 temp_combined = _pext_u64(_mm256_extract_epi64(packed16, 2), pext_mask); + values[1] = temp_combined >> shift2; + if constexpr (BIT_WIDTH != 16) { + values[0] |= static_cast(temp_combined << shift1); } + + const __m256i result = _mm256_setr_epi64x(values[0], values[1], 0, 0); + return result; } else { - out2 |= value << off; + constexpr u32 chunk_bits = BIT_WIDTH * 2; // bits extracted per 64-bit lane + static_assert(chunk_bits < 64); + + constexpr u64 chunk_mask = (chunk_bits == 64) ? ~u64{0} : ((u64{1} << chunk_bits) - 1); + + const u64 x0 = _pext_u64(_mm256_extract_epi64(input, 0), pext_mask) & chunk_mask; + const u64 x1 = _pext_u64(_mm256_extract_epi64(input, 1), pext_mask) & chunk_mask; + const u64 x2 = _pext_u64(_mm256_extract_epi64(input, 2), pext_mask) & chunk_mask; + const u64 x3 = _pext_u64(_mm256_extract_epi64(input, 3), pext_mask) & chunk_mask; + + u64 out0 = 0; + u64 out1 = 0; + u64 out2 = 0; + + auto append_bits = [&](u64 value, u32 bit_offset) { + const u32 word = bit_offset >> 6; // / 64 + const u32 off = bit_offset & 63; // % 64 + + if (word == 0) { + out0 |= value << off; + if (off + chunk_bits > 64) { + out1 |= value >> (64 - off); + } + } else if (word == 1) { + out1 |= value << off; + if (off + chunk_bits > 64) { + out2 |= value >> (64 - off); + } + } else { + out2 |= value << off; + } + }; + + append_bits(x0, 0 * chunk_bits); + append_bits(x1, 1 * chunk_bits); + append_bits(x2, 2 * chunk_bits); + append_bits(x3, 3 * chunk_bits); + + return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), + static_cast(out2), 0); } - }; - - append_bits(x0, 0 * chunk_bits); - append_bits(x1, 1 * chunk_bits); - append_bits(x2, 2 * chunk_bits); - append_bits(x3, 3 * chunk_bits); - - return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), - static_cast(out2), 0); - } -} -} // namespace internal + } + } // namespace internal -/** + /** * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -162,47 +162,47 @@ static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { * * @note This function requires AVX2 and BMI2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const float_t scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_compress_block_bmi2(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - const __m256 scale_v = _mm256_set1_ps(scale); + const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256 source = _mm256_loadu_ps(input); - const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); - const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); - const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); - std::memcpy(output, &packed, BIT_WIDTH); - input += 8; - output += BIT_WIDTH; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256 source = _mm256_loadu_ps(input); + const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); + const __m256i packed_input = internal::mm256_clamp_signed_epi32 < BIT_WIDTH > (quantized); + const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); + std::memcpy(output, &packed, BIT_WIDTH); + input += 8; + output += BIT_WIDTH; + } - if constexpr (remaining) { - std::vector block_values(remaining); + if constexpr (remaining) { + std::vector block_values(remaining); #pragma GCC unroll 8 - for (uint32_t i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized( - internal::quantize_ps_epi32(input[i], scale))); + for (u32 i = 0; i < remaining; i++) { + block_values[i] = + static_cast(internal::clamp_signed_quantized < BIT_WIDTH > ( + internal::quantize_ps_epi32(input[i], scale))); + } + + internal::pack_epi32_fallback < BIT_WIDTH > (block_values, output); } - internal::pack_epi32_fallback(block_values, output); + return 0; } - return 0; -} - -/** + /** * @brief Compress a single block of double values using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -213,50 +213,50 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const float_t * * @note This function requires AVX2 and BMI2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const double_t scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_compress_block_bmi2(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - const __m256d scale_v = _mm256_set1_pd(scale); + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256d source1 = _mm256_loadu_pd(input); - const __m256d source2 = _mm256_loadu_pd(input + 4); - const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); - const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); - __m256i combined = _mm256_castsi128_si256(quantized1); - combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_bmi2( - internal::mm256_clamp_signed_epi32(combined)); - std::memcpy(output, &packed, BIT_WIDTH); - input += 8; - output += BIT_WIDTH; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256d source1 = _mm256_loadu_pd(input); + const __m256d source2 = _mm256_loadu_pd(input + 4); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed = internal::mm256_pack_epi32_bmi2( + internal::mm256_clamp_signed_epi32 < BIT_WIDTH > (combined)); + std::memcpy(output, &packed, BIT_WIDTH); + input += 8; + output += BIT_WIDTH; + } - if constexpr (remaining) { - std::vector block_values(remaining); + if constexpr (remaining) { + std::vector block_values(remaining); #pragma GCC unroll 8 - for (uint32_t i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized( - internal::quantize_pd_epi64(input[i], scale))); - } + for (u32 i = 0; i < remaining; i++) { + block_values[i] = + static_cast(internal::clamp_signed_quantized < BIT_WIDTH > ( + internal::quantize_pd_epi64(input[i], scale))); + } - internal::pack_epi32_fallback(block_values, output); + internal::pack_epi32_fallback < BIT_WIDTH > (block_values, output); + } + return 0; } - return 0; -} -/** + /** * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -269,26 +269,26 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const double_t * * @note This function requires AVX2 and BMI2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const float_t scale, - void* __restrict__ output_ptr, const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const float_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_compress_block_bmi2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_compress_blocks_bmi2(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr, const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const f32 *block_input = input; + u8 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_bmi2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } - return 0; -} + return 0; + } -/** + /** * @brief Compress multiple blocks of double values using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -300,24 +300,24 @@ int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const float_t * * @note This function requires AVX2 and BMI2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const double_t scale, - void* __restrict__ output_ptr, const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const double_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_compress_block_bmi2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_compress_blocks_bmi2(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr, const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const f64 *block_input = input; + u8 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_bmi2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } - return 0; -} + return 0; + } } // namespace pernix #endif // PERNIX_BMI2_COMPRESSION_H diff --git a/src/internal/pernix/x86/bmi2/bmi2_decompression.h b/src/internal/pernix/x86/bmi2/bmi2_decompression.h index 414d6e0..275a11a 100644 --- a/src/internal/pernix/x86/bmi2/bmi2_decompression.h +++ b/src/internal/pernix/x86/bmi2/bmi2_decompression.h @@ -10,48 +10,48 @@ #include namespace pernix { -namespace internal { -/** + namespace internal { + /** * @brief Sign-extend packed values after BMI2 expansion into 32-bit lanes. * * @tparam BIT_WIDTH original encoded bit width. * @param source register containing unpacked values. * @return __m128i sign-extended values. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__m128i mm_sign_extend32(__m128i source) { - if constexpr (BIT_WIDTH == 1) { - // Keep 1-bit values as 0/1 to match fallback decoding semantics. - return source; - } - - constexpr uint16_t shift = 32 - BIT_WIDTH; - source = _mm_slli_epi32(source, shift); - return _mm_srai_epi32(source, shift); -} + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + __m128i mm_sign_extend32(__m128i source) { + if constexpr (BIT_WIDTH == 1) { + // Keep 1-bit values as 0/1 to match fallback decoding semantics. + return source; + } + + constexpr u16 shift = 32 - BIT_WIDTH; + source = _mm_slli_epi32(source, shift); + return _mm_srai_epi32(source, shift); + } -/** + /** * @brief Sign-extend packed values after BMI2 expansion into eight 32-bit lanes. * * @tparam BIT_WIDTH original encoded bit width. * @param source register containing unpacked values. * @return __m256i sign-extended values. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__m256i mm256_sign_extend32(__m256i source) { - if constexpr (BIT_WIDTH == 1) { - // Keep 1-bit values as 0/1 to match fallback decoding semantics. - return source; - } - - constexpr uint16_t shift = 32 - BIT_WIDTH; - source = _mm256_slli_epi32(source, shift); - return _mm256_srai_epi32(source, shift); -} + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + __m256i mm256_sign_extend32(__m256i source) { + if constexpr (BIT_WIDTH == 1) { + // Keep 1-bit values as 0/1 to match fallback decoding semantics. + return source; + } + + constexpr u16 shift = 32 - BIT_WIDTH; + source = _mm256_slli_epi32(source, shift); + return _mm256_srai_epi32(source, shift); + } -/** + /** * @brief Unpack four values from a BMI2-packed input buffer. * * @tparam BIT_WIDTH bit width per packed value. @@ -59,58 +59,58 @@ __m256i mm256_sign_extend32(__m256i source) { * @param input pointer to the packed input buffer. * @return __m128i unpacked values. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH > 0 && BIT_WIDTH <= 24) -__m128i mm_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { - constexpr uint32_t mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; - - __m128i result; - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - constexpr uint64_t pdep_mask = 0x0101010101010101ULL * mask; - - uint32_t temp_value = 0; - std::memcpy(&temp_value, input, packed_bytes); - - const int32_t value = _pdep_u32(temp_value, static_cast(pdep_mask)); - const __m128i source = _mm_insert_epi32(_mm_setzero_si128(), value, 0); - - result = _mm_cvtepi8_epi32(source); - } else if constexpr (BIT_WIDTH == 16) { - const __m128i source = _mm_loadu_si64(input); - result = _mm_cvtepi16_epi32(source); - } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { - constexpr uint64_t pdep_mask = 0x0001000100010001ULL * mask; - - uint64_t temp_value = 0; - std::memcpy(&temp_value, input, packed_bytes); - - const int64_t value = _pdep_u64(temp_value, pdep_mask); - const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); - - result = _mm_cvtepi16_epi32(source); - } else { - constexpr uint64_t pdep_mask = 0x0000000100000001ULL * mask; - constexpr uint32_t shift1 = BIT_WIDTH * 2; - constexpr uint32_t shift2 = 64 - shift1; - - alignas(16) uint64_t temp_values[2]{}; - std::memcpy(temp_values, input, packed_bytes); - - alignas(16) int64_t values[2]; - values[0] = _pdep_u64(temp_values[0], pdep_mask); - values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); - - result = _mm_set_epi64x(values[1], values[0]); - } - - if constexpr (SIGN_VALUES) { - result = internal::mm_sign_extend32(result); - } - return result; -} + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH > 0 && BIT_WIDTH <= 24) + __m128i mm_unpack_epi32_bmi2(const u8 * __restrict__ input) { + constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; + + __m128i result; + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + constexpr u64 pdep_mask = 0x0101010101010101ULL * mask; + + u32 temp_value = 0; + std::memcpy(&temp_value, input, packed_bytes); + + const i32 value = _pdep_u32(temp_value, static_cast(pdep_mask)); + const __m128i source = _mm_insert_epi32(_mm_setzero_si128(), value, 0); + + result = _mm_cvtepi8_epi32(source); + } else if constexpr (BIT_WIDTH == 16) { + const __m128i source = _mm_loadu_si64(input); + result = _mm_cvtepi16_epi32(source); + } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { + constexpr u64 pdep_mask = 0x0001000100010001ULL * mask; + + u64 temp_value = 0; + std::memcpy(&temp_value, input, packed_bytes); + + const i64 value = _pdep_u64(temp_value, pdep_mask); + const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); + + result = _mm_cvtepi16_epi32(source); + } else { + constexpr u64 pdep_mask = 0x0000000100000001ULL * mask; + constexpr u32 shift1 = BIT_WIDTH * 2; + constexpr u32 shift2 = 64 - shift1; + + alignas(16) u64 temp_values[2]{}; + std::memcpy(temp_values, input, packed_bytes); + + alignas(16) i64 values[2]; + values[0] = _pdep_u64(temp_values[0], pdep_mask); + values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); + + result = _mm_set_epi64x(values[1], values[0]); + } + + if constexpr (SIGN_VALUES) { + result = internal::mm_sign_extend32(result); + } + return result; + } -/** + /** * @brief Unpack eight values from a BMI2-packed input buffer. * * @tparam BIT_WIDTH bit width per packed value. @@ -118,80 +118,82 @@ __m128i mm_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { * @param input pointer to the packed input buffer. * @return __m256i unpacked values. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__m256i mm256_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { - constexpr uint32_t mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - constexpr std::size_t packed_bytes = BIT_WIDTH; - - __m256i result; - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - constexpr uint64_t pdep_mask = 0x0101010101010101ULL * mask; - - uint64_t temp_value = 0; - std::memcpy(&temp_value, input, packed_bytes); - - const int64_t value = _pdep_u64(temp_value, pdep_mask); - const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); - - result = _mm256_cvtepi8_epi32(source); - } else if constexpr (BIT_WIDTH == 16) { - const __m128i source = _mm_loadu_si128(reinterpret_cast(input)); - result = _mm256_cvtepi16_epi32(source); - } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { - constexpr uint64_t pdep_mask = 0x0001000100010001ULL * mask; - constexpr uint64_t shift1 = BIT_WIDTH * 4; - constexpr uint64_t shift2 = 64 - shift1; - - alignas(16) uint64_t temp_values[2]{}; - std::memcpy(temp_values, input, packed_bytes); - - alignas(16) int64_t values[2]; - values[0] = _pdep_u64(temp_values[0], pdep_mask); - values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); - - const __m128i source = _mm_set_epi64x(values[1], values[0]); - result = _mm256_cvtepi16_epi32(source); - } else { - constexpr uint64_t pdep_mask = 0x0000000100000001ULL * mask; - constexpr uint32_t shift1 = BIT_WIDTH * 2; - constexpr uint32_t shift2 = 64 - shift1; - - alignas(16) uint64_t temp_values[4]{}; - std::memcpy(temp_values, input, packed_bytes); - - if constexpr ((BIT_WIDTH % 2) == 0) { - std::memcpy(temp_values + 2, input + BIT_WIDTH / 2, packed_bytes - BIT_WIDTH / 2); - } else { - constexpr uint32_t second_group_bit_offset = BIT_WIDTH * 4; - constexpr uint32_t second_group_byte_offset = second_group_bit_offset / 8; - constexpr uint32_t second_group_shift = second_group_bit_offset % 8; - - alignas(16) uint64_t raw_values[2]{}; - std::memcpy(raw_values, input + second_group_byte_offset, packed_bytes - second_group_byte_offset); - - temp_values[2] = (raw_values[0] >> second_group_shift) | (raw_values[1] << (64 - second_group_shift)); - temp_values[3] = raw_values[1] >> second_group_shift; + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + __m256i mm256_unpack_epi32_bmi2(const u8 * __restrict__ input) { + constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + constexpr std::size_t packed_bytes = BIT_WIDTH; + + __m256i result; + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + constexpr u64 pdep_mask = 0x0101010101010101ULL * mask; + + u64 temp_value = 0; + std::memcpy(&temp_value, input, packed_bytes); + + const i64 value = _pdep_u64(temp_value, pdep_mask); + const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); + + result = _mm256_cvtepi8_epi32(source); + } else if constexpr (BIT_WIDTH == 16) { + const __m128i source = _mm_loadu_si128(reinterpret_cast(input)); + result = _mm256_cvtepi16_epi32(source); + } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { + constexpr u64 pdep_mask = 0x0001000100010001ULL * mask; + constexpr u64 shift1 = BIT_WIDTH * 4; + constexpr u64 shift2 = 64 - shift1; + + alignas(16) u64 temp_values[2]{}; + std::memcpy(temp_values, input, packed_bytes); + + alignas(16) i64 values[2]; + values[0] = _pdep_u64(temp_values[0], pdep_mask); + values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); + + const __m128i source = _mm_set_epi64x(values[1], values[0]); + result = _mm256_cvtepi16_epi32(source); + } else { + constexpr u64 pdep_mask = 0x0000000100000001ULL * mask; + constexpr u32 shift1 = BIT_WIDTH * 2; + constexpr u32 shift2 = 64 - shift1; + + alignas(16) u64 temp_values[4]{}; + std::memcpy(temp_values, input, packed_bytes); + + if constexpr ((BIT_WIDTH % 2) == 0) { + std::memcpy(temp_values + 2, input + BIT_WIDTH / 2, packed_bytes - BIT_WIDTH / 2); + } else { + constexpr u32 second_group_bit_offset = BIT_WIDTH * 4; + constexpr u32 second_group_byte_offset = second_group_bit_offset / 8; + constexpr u32 second_group_shift = second_group_bit_offset % 8; + + alignas(16) u64 raw_values[2]{}; + std::memcpy(raw_values, input + second_group_byte_offset, packed_bytes - second_group_byte_offset); + + temp_values[2] = (raw_values[0] >> second_group_shift) | ( + raw_values[1] << (64 - second_group_shift)); + temp_values[3] = raw_values[1] >> second_group_shift; + } + + alignas(16) u64 values[4]; + values[0] = _pdep_u64((temp_values[0]), pdep_mask); + values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); + values[2] = _pdep_u64((temp_values[2]), pdep_mask); + values[3] = _pdep_u64((temp_values[2] >> shift1) | (temp_values[3] << shift2), pdep_mask); + + result = _mm256_set_epi64x(static_cast(values[3]), static_cast(values[2]), + static_cast(values[1]), + static_cast(values[0])); + } + + if constexpr (SIGN_VALUES) { + result = internal::mm256_sign_extend32(result); + } + return result; } + } // namespace internal - alignas(16) uint64_t values[4]; - values[0] = _pdep_u64((temp_values[0]), pdep_mask); - values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); - values[2] = _pdep_u64((temp_values[2]), pdep_mask); - values[3] = _pdep_u64((temp_values[2] >> shift1) | (temp_values[3] << shift2), pdep_mask); - - result = _mm256_set_epi64x(static_cast(values[3]), static_cast(values[2]), static_cast(values[1]), - static_cast(values[0])); - } - - if constexpr (SIGN_VALUES) { - result = internal::mm256_sign_extend32(result); - } - return result; -} -} // namespace internal - -/** + /** * @brief Decompress a single 512\-bit block using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). @@ -205,37 +207,40 @@ __m256i mm256_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { * * @note This function requires AVX2 and BMI2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; - - const __m256 scale_v = _mm256_set1_ps(scale); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_decompress_block_bmi2(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + + const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); - const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); - _mm256_storeu_ps(output, dequantized); - input += BIT_WIDTH; - output += 8; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); + const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); + _mm256_storeu_ps(output, dequantized); + input += BIT_WIDTH; + output += 8; + } - if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (uint32_t i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi32(tail_values[i], scale); + if constexpr (remaining > 0) { + const std::vector tail_values = internal::unpack_epi32_fallback < BIT_WIDTH, SIGN_VALUES + > + (input, remaining); + for (u32 i = 0; i < remaining; i++) { + output[i] = internal::dequantize_epi32(tail_values[i], scale); + } } - } - return 0; -} + return 0; + } -/** + /** * @brief Decompress a single block to double values using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). @@ -249,43 +254,46 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const float_ * * @note This function requires AVX2 and BMI2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; - const __m256d scale_v = _mm256_set1_pd(scale); + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_decompress_block_bmi2(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); - const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); - const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); + const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); + const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); - const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); - const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); + const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); + const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); - _mm256_storeu_pd(output, dequantized1); - _mm256_storeu_pd(output + 4, dequantized2); + _mm256_storeu_pd(output, dequantized1); + _mm256_storeu_pd(output + 4, dequantized2); - input += BIT_WIDTH; - output += 8; - } + input += BIT_WIDTH; + output += 8; + } - if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (uint32_t i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi64(tail_values[i], scale); + if constexpr (remaining > 0) { + const std::vector tail_values = internal::unpack_epi32_fallback < BIT_WIDTH, SIGN_VALUES + > + (input, remaining); + for (u32 i = 0; i < remaining; i++) { + output[i] = internal::dequantize_epi64(tail_values[i], scale); + } } - } - return 0; -} + return 0; + } -/** + /** * @brief Decompress multiple 512\-bit blocks using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -300,26 +308,27 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const double * * @note This function requires AVX2 and BMI2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const float_t scale, void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_decompress_block_bmi2(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_decompress_blocks_bmi2(const void * __restrict__ input_ptr, const f32 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const u8 *block_input = input; + f32 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_decompress_block_bmi2(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } - return 0; -} + return 0; + } -/** + /** * @brief Decompress multiple blocks to double values using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -334,24 +343,25 @@ int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const float * * @note This function requires AVX2 and BMI2 support. */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const double_t scale, void* __restrict__ output_ptr, - const uint32_t blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const uint8_t* block_input = input; - double_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_decompress_block_bmi2(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) + int mm256_decompress_blocks_bmi2(const void * __restrict__ input_ptr, const f64 scale, + void * __restrict__ output_ptr, + const u32 blocks) { + const auto *input = static_cast(input_ptr); + auto *output = static_cast(output_ptr); + + const u8 *block_input = input; + f64 *block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_decompress_block_bmi2(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } - return 0; -} + return 0; + } } // namespace pernix #endif // PERNIX_BMI2_DECOMPRESSION_H diff --git a/src/internal/pernix/x86/utils.h b/src/internal/pernix/x86/utils.h index 185e0a2..42aad76 100644 --- a/src/internal/pernix/x86/utils.h +++ b/src/internal/pernix/x86/utils.h @@ -4,13 +4,11 @@ #include namespace pernix::x86::internal { - -static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t remaining_elements) { - const uint32_t tail_bits = remaining_elements * bit_width; - const uint32_t tail_bytes = (tail_bits + 7u) / 8u; - return tail_bytes; -} - -} // namespace pernix::x86::internal + static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements) { + const u32 tail_bits = remaining_elements * bit_width; + const u32 tail_bytes = (tail_bits + 7u) / 8u; + return tail_bytes; + } +} // namespace pernix::x86::internal #endif // PERNIX_X86_UTILS_H diff --git a/src/pernix.cpp b/src/pernix.cpp index 8b806a6..a57a2e5 100644 --- a/src/pernix.cpp +++ b/src/pernix.cpp @@ -2,14 +2,14 @@ #include namespace { -bool is_valid_block_size(uint32_t block_size) { - return block_size == 64 || block_size == 128 || block_size == 256 || block_size == 512 || block_size == 1024; -} + bool is_valid_block_size(u32 block_size) { + return block_size == 64 || block_size == 128 || block_size == 256 || block_size == 512 || block_size == 1024; + } } extern "C" { -pernix_status pernix_compress_block_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - float scale, void* output) { +pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, + float scale, void *output) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -17,7 +17,8 @@ pernix_status pernix_compress_block_f32(pernix_backend backend, uint8_t bit_widt return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } - const auto kernel = pernix::internal::select_compress_block_f32(static_cast(backend), bit_width, block_size); + const auto kernel = pernix::internal::select_compress_block_f32(static_cast(backend), bit_width, + block_size); if (!kernel) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; @@ -26,8 +27,8 @@ pernix_status pernix_compress_block_f32(pernix_backend backend, uint8_t bit_widt return static_cast(kernel.func(input, scale, output)); } -pernix_status pernix_compress_blocks_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - float scale, void* output, uint32_t blocks) { +pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, + float scale, void *output, u32 blocks) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -36,7 +37,8 @@ pernix_status pernix_compress_blocks_f32(pernix_backend backend, uint8_t bit_wid return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } - const auto kernel = pernix::internal::select_compress_blocks_f32(static_cast(backend), bit_width, block_size); + const auto kernel = pernix::internal::select_compress_blocks_f32(static_cast(backend), bit_width, + block_size); if (!kernel) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; @@ -45,8 +47,8 @@ pernix_status pernix_compress_blocks_f32(pernix_backend backend, uint8_t bit_wid return static_cast(kernel.func(input, scale, output, blocks)); } -pernix_status pernix_decompress_block_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - float scale, void* output, bool sign_values) { +pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, + float scale, void *output, bool sign_values) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -55,8 +57,9 @@ pernix_status pernix_decompress_block_f32(pernix_backend backend, uint8_t bit_wi return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } - const auto kernel = pernix::internal::select_decompress_block_f32(static_cast(backend), bit_width, block_size, - sign_values); + const auto kernel = pernix::internal::select_decompress_block_f32(static_cast(backend), bit_width, + block_size, + sign_values); if (!kernel) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; @@ -65,8 +68,8 @@ pernix_status pernix_decompress_block_f32(pernix_backend backend, uint8_t bit_wi return static_cast(kernel.func(input, scale, output)); } -pernix_status pernix_decompress_blocks_f32(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - float scale, void* output, uint32_t blocks, bool sign_values) { +pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, + float scale, void *output, u32 blocks, bool sign_values) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -75,8 +78,9 @@ pernix_status pernix_decompress_blocks_f32(pernix_backend backend, uint8_t bit_w return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } - const auto kernel = pernix::internal::select_decompress_blocks_f32(static_cast(backend), bit_width, block_size, - sign_values); + const auto kernel = pernix::internal::select_decompress_blocks_f32(static_cast(backend), bit_width, + block_size, + sign_values); if (!kernel) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; @@ -85,8 +89,8 @@ pernix_status pernix_decompress_blocks_f32(pernix_backend backend, uint8_t bit_w return static_cast(kernel.func(input, scale, output, blocks)); } -pernix_status pernix_compress_block_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - double scale, void* output) { +pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, + double scale, void *output) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -95,7 +99,8 @@ pernix_status pernix_compress_block_f64(pernix_backend backend, uint8_t bit_widt return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } - const auto kernel = pernix::internal::select_compress_block_f64(static_cast(backend), bit_width, block_size); + const auto kernel = pernix::internal::select_compress_block_f64(static_cast(backend), bit_width, + block_size); if (!kernel) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; @@ -104,8 +109,8 @@ pernix_status pernix_compress_block_f64(pernix_backend backend, uint8_t bit_widt return static_cast(kernel.func(input, scale, output)); } -pernix_status pernix_compress_blocks_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - double scale, void* output, uint32_t blocks) { +pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, + double scale, void *output, u32 blocks) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -114,7 +119,8 @@ pernix_status pernix_compress_blocks_f64(pernix_backend backend, uint8_t bit_wid return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } - const auto kernel = pernix::internal::select_compress_blocks_f64(static_cast(backend), bit_width, block_size); + const auto kernel = pernix::internal::select_compress_blocks_f64(static_cast(backend), bit_width, + block_size); if (!kernel) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; @@ -123,8 +129,8 @@ pernix_status pernix_compress_blocks_f64(pernix_backend backend, uint8_t bit_wid return static_cast(kernel.func(input, scale, output, blocks)); } -pernix_status pernix_decompress_block_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - double scale, void* output, bool sign_values) { +pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, + double scale, void *output, bool sign_values) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -133,8 +139,9 @@ pernix_status pernix_decompress_block_f64(pernix_backend backend, uint8_t bit_wi return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } - const auto kernel = pernix::internal::select_decompress_block_f64(static_cast(backend), bit_width, block_size, - sign_values); + const auto kernel = pernix::internal::select_decompress_block_f64(static_cast(backend), bit_width, + block_size, + sign_values); if (!kernel) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; @@ -143,8 +150,8 @@ pernix_status pernix_decompress_block_f64(pernix_backend backend, uint8_t bit_wi return static_cast(kernel.func(input, scale, output)); } -pernix_status pernix_decompress_blocks_f64(pernix_backend backend, uint8_t bit_width, uint32_t block_size, const void* input, - double scale, void* output, uint32_t blocks, bool sign_values) { +pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, + double scale, void *output, u32 blocks, bool sign_values) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -153,8 +160,9 @@ pernix_status pernix_decompress_blocks_f64(pernix_backend backend, uint8_t bit_w return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } - const auto kernel = pernix::internal::select_decompress_blocks_f64(static_cast(backend), bit_width, block_size, - sign_values); + const auto kernel = pernix::internal::select_decompress_blocks_f64(static_cast(backend), bit_width, + block_size, + sign_values); if (!kernel) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; diff --git a/src/x86/avx2/avx2_compression.cpp b/src/x86/avx2/avx2_compression.cpp index 1a7f2c0..548989b 100644 --- a/src/x86/avx2/avx2_compression.cpp +++ b/src/x86/avx2/avx2_compression.cpp @@ -134,53 +134,53 @@ case N: return Kernel("avx2", &mm256_compress_blocks_avx2 select_avx2_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"avx2", nullptr}; + Kernel select_avx2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } } -} -Kernel select_avx2_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"avx2", nullptr}; + Kernel select_avx2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } } -} -Kernel select_avx2_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"avx2", nullptr}; + Kernel select_avx2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } } -} -Kernel select_avx2_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"avx2", nullptr}; + Kernel select_avx2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } } -} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 diff --git a/src/x86/avx2/avx2_decompression.cpp b/src/x86/avx2/avx2_decompression.cpp index 43558ae..cf8a53c 100644 --- a/src/x86/avx2/avx2_decompression.cpp +++ b/src/x86/avx2/avx2_decompression.cpp @@ -146,49 +146,53 @@ case N: \ } \ break -Kernel select_avx2_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"avx2", nullptr}; + Kernel select_avx2_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"avx2", nullptr}; + } } -} -Kernel select_avx2_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"avx2", nullptr}; + Kernel select_avx2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"avx2", nullptr}; + } } -} -Kernel select_avx2_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"avx2", nullptr}; + Kernel select_avx2_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"avx2", nullptr}; + } } -} -Kernel select_avx2_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"avx2", nullptr}; + Kernel select_avx2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"avx2", nullptr}; + } } -} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 diff --git a/src/x86/avx512vbmi/avx512vbmi_compression.cpp b/src/x86/avx512vbmi/avx512vbmi_compression.cpp index f1c36a1..8eb25c5 100644 --- a/src/x86/avx512vbmi/avx512vbmi_compression.cpp +++ b/src/x86/avx512vbmi/avx512vbmi_compression.cpp @@ -134,53 +134,53 @@ case N: return Kernel("avx512vbmi", &mm512_compress_blocks_ default: return {"avx512vbmi", nullptr}; \ } -Kernel select_avx512vbmi_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"avx512vbmi", nullptr}; + Kernel select_avx512vbmi_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } } -} -Kernel select_avx512vbmi_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"avx512vbmi", nullptr}; + Kernel select_avx512vbmi_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } } -} -Kernel select_avx512vbmi_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"avx512vbmi", nullptr}; + Kernel select_avx512vbmi_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } } -} -Kernel select_avx512vbmi_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"avx512vbmi", nullptr}; + Kernel select_avx512vbmi_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } } -} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 diff --git a/src/x86/avx512vbmi/avx512vbmi_decompression.cpp b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp index 43f45fc..ac2cbcd 100644 --- a/src/x86/avx512vbmi/avx512vbmi_decompression.cpp +++ b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp @@ -146,49 +146,53 @@ case N: \ } \ break -Kernel select_avx512vbmi_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"avx512vbmi", nullptr}; + Kernel select_avx512vbmi_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"avx512vbmi", nullptr}; + } } -} -Kernel select_avx512vbmi_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"avx512vbmi", nullptr}; + Kernel select_avx512vbmi_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"avx512vbmi", nullptr}; + } } -} -Kernel select_avx512vbmi_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"avx512vbmi", nullptr}; + Kernel select_avx512vbmi_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"avx512vbmi", nullptr}; + } } -} -Kernel select_avx512vbmi_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"avx512vbmi", nullptr}; + Kernel select_avx512vbmi_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"avx512vbmi", nullptr}; + } } -} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 diff --git a/src/x86/bmi2/bmi2_compression.cpp b/src/x86/bmi2/bmi2_compression.cpp index ea01e04..e978ba2 100644 --- a/src/x86/bmi2/bmi2_compression.cpp +++ b/src/x86/bmi2/bmi2_compression.cpp @@ -134,53 +134,53 @@ case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2 select_bmi2_compress_block_f32(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"bmi2", nullptr}; + Kernel select_bmi2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } } -} -Kernel select_bmi2_compress_blocks_f32(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"bmi2", nullptr}; + Kernel select_bmi2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } } -} -Kernel select_bmi2_compress_block_f64(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"bmi2", nullptr}; + Kernel select_bmi2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } } -} -Kernel select_bmi2_compress_blocks_f64(const uint8_t bit_width, const uint32_t block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"bmi2", nullptr}; + Kernel select_bmi2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } } -} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 diff --git a/src/x86/bmi2/bmi2_decompression.cpp b/src/x86/bmi2/bmi2_decompression.cpp index e829c24..84deda8 100644 --- a/src/x86/bmi2/bmi2_decompression.cpp +++ b/src/x86/bmi2/bmi2_decompression.cpp @@ -146,49 +146,53 @@ case N: \ } \ break -Kernel select_bmi2_decompress_block_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"bmi2", nullptr}; + Kernel select_bmi2_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"bmi2", nullptr}; + } } -} -Kernel select_bmi2_decompress_blocks_f32(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"bmi2", nullptr}; + Kernel select_bmi2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"bmi2", nullptr}; + } } -} -Kernel select_bmi2_decompress_block_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"bmi2", nullptr}; + Kernel select_bmi2_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"bmi2", nullptr}; + } } -} -Kernel select_bmi2_decompress_blocks_f64(const uint8_t bit_width, const uint32_t block_size, bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"bmi2", nullptr}; + Kernel select_bmi2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"bmi2", nullptr}; + } } -} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp index 5e2b4ef..abb502b 100644 --- a/tests/fallback_tests.cpp +++ b/tests/fallback_tests.cpp @@ -12,9 +12,9 @@ // --------------------------------------------------------------------------- TYPED_TEST(CompressionTest, FallbackCompressBlock) { - std::vector> compressed(this->testSet.numberOfBlocks); + std::vector > compressed(this->testSet.numberOfBlocks); - for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { compressed[b].resize(TestFixture::BlockSize); const auto status = pernix_compress_block_f32( @@ -24,15 +24,15 @@ TYPED_TEST(CompressionTest, FallbackCompressBlock) { ASSERT_EQ(status, PERNIX_STATUS_OK); } - for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { expectCompressedBlockEqualsReference(*this, compressed[b], b); } } TYPED_TEST(CompressionTest64, FallbackCompressBlock) { - std::vector> compressed(this->testSet.numberOfBlocks); + std::vector > compressed(this->testSet.numberOfBlocks); - for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { compressed[b].resize(TestFixture::BlockSize); const auto status = pernix_compress_block_f64( @@ -42,7 +42,7 @@ TYPED_TEST(CompressionTest64, FallbackCompressBlock) { ASSERT_EQ(status, PERNIX_STATUS_OK); } - for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { expectCompressedBlockEqualsReference(*this, compressed[b], b); } } @@ -52,9 +52,9 @@ TYPED_TEST(CompressionTest64, FallbackCompressBlock) { // --------------------------------------------------------------------------- TYPED_TEST(DecompressionTest, FallbackDecompressBlock) { - std::vector> decompressed(this->testSet.numberOfBlocks); + std::vector > decompressed(this->testSet.numberOfBlocks); - for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { decompressed[b].resize(this->testSet.elementsPerBlock); const auto status = pernix_decompress_block_f32( @@ -64,15 +64,15 @@ TYPED_TEST(DecompressionTest, FallbackDecompressBlock) { ASSERT_EQ(status, PERNIX_STATUS_OK); } - for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { expectDecompressedBlockNearSource(*this, decompressed[b], b); } } TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { - std::vector> decompressed(this->testSet.numberOfBlocks); + std::vector > decompressed(this->testSet.numberOfBlocks); - for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { decompressed[b].resize(this->testSet.elementsPerBlock); const auto status = pernix_decompress_block_f64( @@ -82,7 +82,7 @@ TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { ASSERT_EQ(status, PERNIX_STATUS_OK); } - for (uint32_t b = 0; b < this->testSet.numberOfBlocks; b++) { + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { expectDecompressedBlockNearSource(*this, decompressed[b], b); } } @@ -92,77 +92,77 @@ TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { // --------------------------------------------------------------------------- TYPED_TEST(CompressionTest, FallbackCompressBlocksRoundtrip) { - const uint32_t nb = this->testSet.numberOfBlocks; - const uint32_t epb = this->testSet.elementsPerBlock; - const uint32_t total = nb * epb; + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; + const u32 total = nb * epb; - std::vector flat(total); - for (uint32_t b = 0; b < nb; b++) { + std::vector flat(total); + for (u32 b = 0; b < nb; b++) { std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); } // Compute a single scale that covers all blocks float max_abs = 0.0f; - for (uint32_t i = 0; i < total; i++) { + for (u32 i = 0; i < total; i++) { max_abs = std::max(max_abs, std::abs(flat[i])); } const float q = static_cast(decltype(this->testSet)::quantization_levels); - const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); + const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); const float scale_inv = 1.0f / scale; - std::vector compressed(nb * TestFixture::BlockSize); + std::vector compressed(nb * TestFixture::BlockSize); auto status = pernix_compress_blocks_f32( PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), scale_inv, compressed.data(), nb); ASSERT_EQ(status, PERNIX_STATUS_OK); - std::vector restored(total); + std::vector restored(total); status = pernix_decompress_blocks_f32( PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, restored.data(), nb, true); ASSERT_EQ(status, PERNIX_STATUS_OK); - const float tol = (std::abs(scale) * 0.5f) + (std::numeric_limits::epsilon() * 16.0f); - for (uint32_t i = 0; i < total; i++) { + const float tol = (std::abs(scale) * 0.5f) + (std::numeric_limits::epsilon() * 16.0f); + for (u32 i = 0; i < total; i++) { EXPECT_NEAR(restored[i], flat[i], tol); } } TYPED_TEST(CompressionTest64, FallbackCompressBlocksRoundtrip) { - const uint32_t nb = this->testSet.numberOfBlocks; - const uint32_t epb = this->testSet.elementsPerBlock; - const uint32_t total = nb * epb; + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; + const u32 total = nb * epb; - std::vector flat(total); - for (uint32_t b = 0; b < nb; b++) { + std::vector flat(total); + for (u32 b = 0; b < nb; b++) { std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); } // Compute a single scale that covers all blocks double max_abs = 0.0; - for (uint32_t i = 0; i < total; i++) { + for (u32 i = 0; i < total; i++) { max_abs = std::max(max_abs, std::abs(flat[i])); } const double q = static_cast(decltype(this->testSet)::quantization_levels); - const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); + const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); const double scale_inv = 1.0 / scale; - std::vector compressed(nb * TestFixture::BlockSize); + std::vector compressed(nb * TestFixture::BlockSize); auto status = pernix_compress_blocks_f64( PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), scale_inv, compressed.data(), nb); ASSERT_EQ(status, PERNIX_STATUS_OK); - std::vector restored(total); + std::vector restored(total); status = pernix_decompress_blocks_f64( PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, restored.data(), nb, true); ASSERT_EQ(status, PERNIX_STATUS_OK); - const double tol = (std::abs(scale) * 0.5) + (std::numeric_limits::epsilon() * 16.0); - for (uint32_t i = 0; i < total; i++) { + const double tol = (std::abs(scale) * 0.5) + (std::numeric_limits::epsilon() * 16.0); + for (u32 i = 0; i < total; i++) { EXPECT_NEAR(restored[i], flat[i], tol); } } @@ -172,11 +172,11 @@ TYPED_TEST(CompressionTest64, FallbackCompressBlocksRoundtrip) { // --------------------------------------------------------------------------- TYPED_TEST(CompressionTest, SingleBlockCompressBlocksMatchesBlock) { - const auto& src = this->testSet.getDecompressedData()[0]; + const auto &src = this->testSet.getDecompressedData()[0]; const float scale_inv = 1.0f / this->testSet.getScales()[0]; - std::vector blockOut(TestFixture::BlockSize); - std::vector blocksOut(TestFixture::BlockSize); + std::vector blockOut(TestFixture::BlockSize); + std::vector blocksOut(TestFixture::BlockSize); auto s1 = pernix_compress_block_f32( PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, @@ -188,18 +188,18 @@ TYPED_TEST(CompressionTest, SingleBlockCompressBlocksMatchesBlock) { src.data(), scale_inv, blocksOut.data(), 1); ASSERT_EQ(s2, PERNIX_STATUS_OK); - for (uint32_t i = 0; i < TestFixture::BlockSize; i++) { + for (u32 i = 0; i < TestFixture::BlockSize; i++) { EXPECT_EQ(blockOut[i], blocksOut[i]) << "byte " << i; } } TYPED_TEST(DecompressionTest, SingleBlockDecompressBlocksMatchesBlock) { - const auto& compressed = this->testSet.getCompressedData()[0]; + const auto &compressed = this->testSet.getCompressedData()[0]; const float scale = this->testSet.getScales()[0]; - const uint32_t epb = this->testSet.elementsPerBlock; + const u32 epb = this->testSet.elementsPerBlock; - std::vector blockOut(epb); - std::vector blocksOut(epb); + std::vector blockOut(epb); + std::vector blocksOut(epb); auto s1 = pernix_decompress_block_f32( PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, @@ -211,7 +211,7 @@ TYPED_TEST(DecompressionTest, SingleBlockDecompressBlocksMatchesBlock) { compressed.data(), scale, blocksOut.data(), 1, true); ASSERT_EQ(s2, PERNIX_STATUS_OK); - for (uint32_t i = 0; i < epb; i++) { + for (u32 i = 0; i < epb; i++) { EXPECT_FLOAT_EQ(blockOut[i], blocksOut[i]) << "element " << i; } } @@ -221,11 +221,11 @@ TYPED_TEST(DecompressionTest, SingleBlockDecompressBlocksMatchesBlock) { // --------------------------------------------------------------------------- TEST(FallbackEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { - constexpr uint32_t BS = 64; - const std::array input{0x08}; + constexpr u32 BS = 64; + const std::array input{0x08}; pernix_status st; - std::array output{}; + std::array output{}; st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, output.data(), true); ASSERT_EQ(st, PERNIX_STATUS_OK); @@ -233,12 +233,12 @@ TEST(FallbackEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { } TEST(FallbackEdgeTest, ClearsUnusedPaddingBytes) { - constexpr uint32_t BS = 64; - constexpr uint32_t BW = 24; - constexpr uint32_t EPB = (BS * 8) / BW; + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; - std::array input{}; - std::array output{}; + std::array input{}; + std::array output{}; output.fill(0xA5); auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, output.data()); @@ -247,17 +247,17 @@ TEST(FallbackEdgeTest, ClearsUnusedPaddingBytes) { } TEST(FallbackEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { - constexpr uint32_t BS = 64; - constexpr uint32_t BW = 4; - constexpr uint32_t EPB = (BS * 8) / BW; + constexpr u32 BS = 64; + constexpr u32 BW = 4; + constexpr u32 EPB = (BS * 8) / BW; - std::array input{}; - input[0] = std::numeric_limits::infinity(); - input[1] = -std::numeric_limits::infinity(); - input[2] = std::numeric_limits::quiet_NaN(); + std::array input{}; + input[0] = std::numeric_limits::infinity(); + input[1] = -std::numeric_limits::infinity(); + input[2] = std::numeric_limits::quiet_NaN(); - std::array compressed{}; - std::array restored{}; + std::array compressed{}; + std::array restored{}; auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, compressed.data()); ASSERT_EQ(st, PERNIX_STATUS_OK); @@ -275,9 +275,9 @@ TEST(FallbackEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { // --------------------------------------------------------------------------- TEST(ErrorCodeTest, UnsupportedBlockSizeReturnsError) { - constexpr uint32_t BS = 32; - float_t src[32] = {}; - uint8_t dst[32] = {}; + constexpr u32 BS = 32; + f32 src[32] = {}; + u8 dst[32] = {}; auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, BS, src, 1.0f, dst); EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); @@ -293,9 +293,9 @@ TEST(ErrorCodeTest, UnsupportedBlockSizeReturnsError) { } TEST(ErrorCodeTest, UnsupportedBitWidthReturnsError) { - constexpr uint32_t BS = 64; - float_t src[256] = {}; - uint8_t dst[64] = {}; + constexpr u32 BS = 64; + f32 src[256] = {}; + u8 dst[64] = {}; auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, BS, src, 1.0f, dst); EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); @@ -305,8 +305,8 @@ TEST(ErrorCodeTest, UnsupportedBitWidthReturnsError) { } TEST(ErrorCodeTest, NullPointerReturnsError) { - float_t src[64] = {}; - uint8_t dst[64] = {}; + f32 src[64] = {}; + u8 dst[64] = {}; auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, 64, nullptr, 1.0f, dst); EXPECT_EQ(st, PERNIX_STATUS_INVALID_ARGUMENT); diff --git a/tests/include/testset.h b/tests/include/testset.h index 81acb32..b73fc71 100644 --- a/tests/include/testset.h +++ b/tests/include/testset.h @@ -21,11 +21,11 @@ static_assert(PERNIX_TEST_BLOCK_SIZE % 32 == 0, "PERNIX_TEST_BLOCK_SIZE must be dividable by 32 bytes"); -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && std::is_floating_point_v class TestSet { - using ValueType = uint8_t; - using SeedType = std::mt19937::result_type; + using ValueType = u8; + using SeedType = std::mt19937::result_type; alignas(64) std::vector > compressedData; alignas(64) std::vector > sourceData; @@ -36,21 +36,24 @@ class TestSet { std::uniform_real_distribution dis{}; public: - static constexpr uint32_t elementsPerBlock = (BLOCK_SIZE * 8) / BIT_WIDTH; - static constexpr SeedType defaultSeed = 0x5eed1234u; + static constexpr u32 elementsPerBlock = (BLOCK_SIZE * 8) / BIT_WIDTH; + static constexpr SeedType defaultSeed = 0x5eed1234u; static constexpr T quantization_levels = - SIGN_VALUES ? static_cast(BIT_WIDTH == 1 ? 1u : ((1u << (BIT_WIDTH - 1u)) - 1u)) : static_cast((1u << BIT_WIDTH) - 1u); + SIGN_VALUES + ? static_cast(BIT_WIDTH == 1 ? 1u : ((1u << (BIT_WIDTH - 1u)) - 1u)) + : static_cast((1u << BIT_WIDTH) - 1u); - uint32_t numberOfBlocks; + u32 numberOfBlocks; - [[nodiscard]] constexpr uint32_t totalElements() const { return numberOfBlocks * elementsPerBlock; } + [[nodiscard]] constexpr u32 totalElements() const { return numberOfBlocks * elementsPerBlock; } - [[nodiscard]] T blockTolerance(const uint32_t block) const { - return (std::abs(scalesData[block]) * static_cast(0.5)) + (std::numeric_limits::epsilon() * static_cast(16)); + [[nodiscard]] T blockTolerance(const u32 block) const { + return (std::abs(scalesData[block]) * static_cast(0.5)) + ( + std::numeric_limits::epsilon() * static_cast(16)); } - explicit TestSet(const uint32_t number_of_blocks, const SeedType initial_seed = testSeed()) + explicit TestSet(const u32 number_of_blocks, const SeedType initial_seed = testSeed()) : seed(initial_seed), gen(seed), numberOfBlocks(number_of_blocks) { compressedData.resize(numberOfBlocks); sourceData.resize(number_of_blocks); @@ -59,32 +62,32 @@ class TestSet { generateData(); } - [[nodiscard]] const std::vector& getScales() const { return scalesData; } + [[nodiscard]] const std::vector &getScales() const { return scalesData; } - [[nodiscard]] const std::vector >& getCompressedData() const { return compressedData; } + [[nodiscard]] const std::vector > &getCompressedData() const { return compressedData; } - [[nodiscard]] const std::vector >& getDecompressedData() const { return sourceData; } + [[nodiscard]] const std::vector > &getDecompressedData() const { return sourceData; } [[nodiscard]] SeedType getSeed() const { return seed; } [[nodiscard]] static SeedType testSeed() { - const char* env_seed = std::getenv("PERNIX_TEST_SEED"); + const char *env_seed = std::getenv("PERNIX_TEST_SEED"); if (env_seed == nullptr || *env_seed == '\0') { return defaultSeed; } - char* end = nullptr; + char *end = nullptr; const unsigned long value = std::strtoul(env_seed, &end, 0); return (end != env_seed && *end == '\0') ? static_cast(value) : defaultSeed; } private: void generateData() { - for (uint32_t i = 0; i < numberOfBlocks; i++) { + for (u32 i = 0; i < numberOfBlocks; i++) { compressedData[i].resize(BLOCK_SIZE); sourceData[i].resize(elementsPerBlock); - for (uint32_t j = 0; j < elementsPerBlock; j++) { + for (u32 j = 0; j < elementsPerBlock; j++) { sourceData[i][j] = dis(gen); } @@ -97,133 +100,138 @@ class TestSet { if constexpr (std::is_same_v) { pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, - sourceData[i].data(), 1.0f / scalesData[i], compressedData[i].data()); + sourceData[i].data(), 1.0f / scalesData[i], compressedData[i].data()); } else { pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, - sourceData[i].data(), 1.0 / scalesData[i], compressedData[i].data()); + sourceData[i].data(), 1.0 / scalesData[i], compressedData[i].data()); } } } }; -template +template struct BitWidthBlockSize { - static constexpr uint8_t bit_width = BIT_WIDTH; - static constexpr uint32_t block_size = BLOCK_SIZE; + static constexpr u8 bit_width = BIT_WIDTH; + static constexpr u32 block_size = BLOCK_SIZE; }; using testing::Types; using BitWidthBlockSizeTypes = Types, BitWidthBlockSize<2, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<3, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<4, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<5, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<6, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<7, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<8, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<9, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<10, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<11, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<12, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<13, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<14, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<15, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<16, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<17, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<18, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<19, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<20, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<21, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<22, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<23, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<24, PERNIX_TEST_BLOCK_SIZE> >; + BitWidthBlockSize<3, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<4, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<5, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<6, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<7, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<8, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<9, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<10, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<11, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<12, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<13, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<14, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<15, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<16, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<17, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<18, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<19, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<20, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<21, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<22, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<23, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<24, PERNIX_TEST_BLOCK_SIZE> >; struct BitWidthBlockSizeName { - template + template static std::string GetName(int) { std::ostringstream name; - name << "BitWidth" << static_cast(TestConfigT::bit_width) << "BlockSize" << TestConfigT::block_size; + name << "BitWidth" << static_cast(TestConfigT::bit_width) << "BlockSize" << TestConfigT::block_size; return name.str(); } }; -template +template class CompressionTest : public ::testing::Test { public: - static constexpr uint8_t BitWidth = TestConfigT::bit_width; - static constexpr uint32_t BlockSize = TestConfigT::block_size; + static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u32 BlockSize = TestConfigT::block_size; - TestSet testSet; + TestSet testSet; CompressionTest() : testSet(1u << 10) { } }; -template +template class DecompressionTest : public ::testing::Test { public: - static constexpr uint8_t BitWidth = TestConfigT::bit_width; - static constexpr uint32_t BlockSize = TestConfigT::block_size; + static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u32 BlockSize = TestConfigT::block_size; - TestSet testSet; + TestSet testSet; DecompressionTest() : testSet(1u << 10) { } }; -template +template class CompressionTest64 : public ::testing::Test { public: - static constexpr uint8_t BitWidth = TestConfigT::bit_width; - static constexpr uint32_t BlockSize = TestConfigT::block_size; + static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u32 BlockSize = TestConfigT::block_size; - TestSet testSet; + TestSet testSet; CompressionTest64() : testSet(1u << 10) { } }; -template +template class DecompressionTest64 : public ::testing::Test { public: - static constexpr uint8_t BitWidth = TestConfigT::bit_width; - static constexpr uint32_t BlockSize = TestConfigT::block_size; + static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u32 BlockSize = TestConfigT::block_size; - TestSet testSet; + TestSet testSet; DecompressionTest64() : testSet(1u << 10) { } }; -template -[[nodiscard]] std::string testContext(const FixtureT& fixture, const uint32_t block) { +template +[[nodiscard]] std::string testContext(const FixtureT &fixture, const u32 block) { std::ostringstream message; - message << "bit_width=" << static_cast(FixtureT::BitWidth) << ", block_size=" << FixtureT::BlockSize - << ", block=" << block << ", scale=" << fixture.testSet.getScales()[block] - << ", tolerance=" << fixture.testSet.blockTolerance(block) << ", seed=" << fixture.testSet.getSeed(); + message << "bit_width=" << static_cast(FixtureT::BitWidth) << ", block_size=" << FixtureT::BlockSize + << ", block=" << block << ", scale=" << fixture.testSet.getScales()[block] + << ", tolerance=" << fixture.testSet.blockTolerance(block) << ", seed=" << fixture.testSet.getSeed(); return message.str(); } -template -void expectCompressedBlockEqualsReference(const FixtureT& fixture, const std::vector& actual, const uint32_t block) { +template +void expectCompressedBlockEqualsReference(const FixtureT &fixture, const std::vector &actual, + const u32 block) { SCOPED_TRACE(testContext(fixture, block)); - const auto& expected = fixture.testSet.getCompressedData()[block]; + const auto &expected = fixture.testSet.getCompressedData()[block]; ASSERT_EQ(actual.size(), expected.size()) << "Compressed block byte count differs from reference"; - for (uint32_t byte = 0; byte < actual.size(); byte++) { + for (u32 byte = 0; byte < actual.size(); byte++) { ASSERT_EQ(actual[byte], expected[byte]) - << "Compressed byte mismatch at byte=" << byte << ", actual=" << static_cast(actual[byte]) - << ", expected=" << static_cast(expected[byte]); + << "Compressed byte mismatch at byte=" << byte << ", actual=" << static_cast(actual[byte]) + << ", expected=" << static_cast(expected[byte]); } } -template -void expectDecompressedBlockNearSource(const FixtureT& fixture, const std::vector& actual, const uint32_t block) { +template +void expectDecompressedBlockNearSource(const FixtureT &fixture, const std::vector &actual, const u32 block) { SCOPED_TRACE(testContext(fixture, block)); - const auto& expected = fixture.testSet.getDecompressedData()[block]; + const auto &expected = fixture.testSet.getDecompressedData()[block]; ASSERT_EQ(actual.size(), expected.size()) << "Decompressed block element count differs from source"; - for (uint32_t element = 0; element < actual.size(); element++) { + for (u32 element = 0; element < actual.size(); element++) { ASSERT_NEAR(actual[element], expected[element], fixture.testSet.blockTolerance(block)) - << "Decompressed element mismatch at element=" << element << ", actual=" << actual[element] - << ", expected=" << expected[element] << ", absolute_error=" << std::abs(actual[element] - expected[element]); + << "Decompressed element mismatch at element=" << element << ", actual=" << actual[element] + << ", expected=" << expected[element] << ", absolute_error=" << std::abs( + actual[element] - expected[element]); } } TYPED_TEST_SUITE(CompressionTest, BitWidthBlockSizeTypes, BitWidthBlockSizeName); + TYPED_TEST_SUITE(DecompressionTest, BitWidthBlockSizeTypes, BitWidthBlockSizeName); + TYPED_TEST_SUITE(CompressionTest64, BitWidthBlockSizeTypes, BitWidthBlockSizeName); + TYPED_TEST_SUITE(DecompressionTest64, BitWidthBlockSizeTypes, BitWidthBlockSizeName); #endif // PERNIX_TESTSET_H diff --git a/tests/simd_tests.cpp b/tests/simd_tests.cpp index 6f9bc50..019aea3 100644 --- a/tests/simd_tests.cpp +++ b/tests/simd_tests.cpp @@ -7,50 +7,50 @@ // SIMD compress: compress via backend, decompress via fallback, compare source // --------------------------------------------------------------------------- -template -void testBackendCompressBlock(FixtureT& fixture, pernix_backend backend) { +template +void testBackendCompressBlock(FixtureT &fixture, pernix_backend backend) { using T = std::remove_cvref_t; { - std::vector probe(FixtureT::BlockSize); + std::vector probe(FixtureT::BlockSize); pernix_status st; if constexpr (std::is_same_v) { st = pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getDecompressedData()[0].data(), - 1.0f / fixture.testSet.getScales()[0], probe.data()); + fixture.testSet.getDecompressedData()[0].data(), + 1.0f / fixture.testSet.getScales()[0], probe.data()); } else { st = pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getDecompressedData()[0].data(), - 1.0 / fixture.testSet.getScales()[0], probe.data()); + fixture.testSet.getDecompressedData()[0].data(), + 1.0 / fixture.testSet.getScales()[0], probe.data()); } if (st != PERNIX_STATUS_OK) { - SUCCEED(); + GTEST_SKIP(); return; } } - for (uint32_t b = 0; b < fixture.testSet.numberOfBlocks; b++) { - std::vector compressed(FixtureT::BlockSize); + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; b++) { + std::vector compressed(FixtureT::BlockSize); pernix_status st; if constexpr (std::is_same_v) { st = pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getDecompressedData()[b].data(), - 1.0f / fixture.testSet.getScales()[b], compressed.data()); + fixture.testSet.getDecompressedData()[b].data(), + 1.0f / fixture.testSet.getScales()[b], compressed.data()); } else { st = pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getDecompressedData()[b].data(), - 1.0 / fixture.testSet.getScales()[b], compressed.data()); + fixture.testSet.getDecompressedData()[b].data(), + 1.0 / fixture.testSet.getScales()[b], compressed.data()); } ASSERT_EQ(st, PERNIX_STATUS_OK); std::vector restored(fixture.testSet.elementsPerBlock); if constexpr (std::is_same_v) { st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, - compressed.data(), fixture.testSet.getScales()[b], restored.data(), true); + compressed.data(), fixture.testSet.getScales()[b], restored.data(), true); } else { st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, - compressed.data(), fixture.testSet.getScales()[b], restored.data(), true); + compressed.data(), fixture.testSet.getScales()[b], restored.data(), true); } ASSERT_EQ(st, PERNIX_STATUS_OK); @@ -62,8 +62,8 @@ void testBackendCompressBlock(FixtureT& fixture, pernix_backend backend) { // SIMD decompress: decompress fallback-compressed data via backend, compare source // --------------------------------------------------------------------------- -template -void testBackendDecompressBlock(FixtureT& fixture, pernix_backend backend) { +template +void testBackendDecompressBlock(FixtureT &fixture, pernix_backend backend) { using T = std::remove_cvref_t; { @@ -71,31 +71,31 @@ void testBackendDecompressBlock(FixtureT& fixture, pernix_backend backend) { pernix_status st; if constexpr (std::is_same_v) { st = pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getCompressedData()[0].data(), - fixture.testSet.getScales()[0], probe.data(), true); + fixture.testSet.getCompressedData()[0].data(), + fixture.testSet.getScales()[0], probe.data(), true); } else { st = pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getCompressedData()[0].data(), - fixture.testSet.getScales()[0], probe.data(), true); + fixture.testSet.getCompressedData()[0].data(), + fixture.testSet.getScales()[0], probe.data(), true); } if (st != PERNIX_STATUS_OK) { - SUCCEED(); + GTEST_SKIP(); return; } } - for (uint32_t b = 0; b < fixture.testSet.numberOfBlocks; b++) { + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; b++) { std::vector decompressed(fixture.testSet.elementsPerBlock); pernix_status st; if constexpr (std::is_same_v) { st = pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getCompressedData()[b].data(), - fixture.testSet.getScales()[b], decompressed.data(), true); + fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed.data(), true); } else { st = pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getCompressedData()[b].data(), - fixture.testSet.getScales()[b], decompressed.data(), true); + fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed.data(), true); } ASSERT_EQ(st, PERNIX_STATUS_OK); From bdf3168842705bcb47f485c841498552987e8f73 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Sat, 27 Jun 2026 15:40:57 +0200 Subject: [PATCH 15/43] refactor: rename internal headers and add fallback compression/decompression implementations --- CMakeLists.txt | 3 + .../pernix/arm64/neon/common.h | 0 .../pernix/arm64/neon/compression.h | 0 .../pernix/arm64/neon/decompression.h | 0 .../pernix/arm64/neon/packing.h | 0 .../pernix/arm64/neon/tables.h | 0 .../pernix/arm64/neon/unpacking.h | 0 .../pernix/arm64/sve2/compression.h | 0 .../pernix/arm64/sve2/decompression.h | 0 .../pernix/arm64/sve2/packing.h | 0 .../pernix/arm64/sve2/tables.h | 0 .../pernix/arm64/sve2/unpacking.h | 0 include/pernix/backend.hpp | 21 + include/pernix/detail/api.hpp | 124 + include/pernix/dispatch/cpu_features.h | 113 + .../pernix/dispatch/kernel.h | 0 .../pernix/dispatch/select.h | 94 +- include/pernix/dispatch/select_impl.h | 3210 +++++++++++++++++ .../pernix/fallback/scalar_compression.h | 97 +- .../pernix/fallback/scalar_decompression.h | 95 +- include/pernix/fallback/simd_compression.h | 47 + include/pernix/fallback/simd_decompression.h | 51 + include/pernix/fallback/stdpar_compression.h | 40 + .../pernix/fallback/stdpar_decompression.h | 44 + include/pernix/pernix.h | 9 +- include/pernix/pernix.hpp | 48 +- .../internal => include}/pernix/simd_compat.h | 0 .../pernix/x86/avx2/avx2_compression.h | 2 +- .../pernix/x86/avx2/avx2_decompression.h | 2 +- .../pernix/x86/avx2/avx2_tables.h | 0 .../x86/avx512vbmi/avx512vbmi_compression.h | 0 .../x86/avx512vbmi/avx512vbmi_decompression.h | 0 .../pernix/x86/avx512vbmi/compat.h | 0 .../pernix/x86/avx512vbmi/packing.h | 0 .../pernix/x86/avx512vbmi/tables.h | 0 .../pernix/x86/avx512vbmi/unpacking.h | 0 .../pernix/x86/bmi2/bmi2_compression.h | 2 +- .../pernix/x86/bmi2/bmi2_decompression.h | 0 {src/internal => include}/pernix/x86/utils.h | 0 src/CMakeLists.txt | 291 +- src/dispatch/select.cpp | 193 + src/fallback/fallback_compression.cpp | 4 +- src/fallback/fallback_decompression.cpp | 2 +- src/fallback/simd_compression.cpp | 4 + src/fallback/simd_decompression.cpp | 4 + src/fallback/stdpar_compression.cpp | 4 + src/fallback/stdpar_decompression.cpp | 4 + src/internal/pernix/dispatch/cpu_features.h | 26 - src/pernix.cpp | 156 +- tests/CMakeLists.txt | 15 + tests/fallback_tests.cpp | 81 + tests/header_only_tests.cpp | 69 + 52 files changed, 4240 insertions(+), 615 deletions(-) rename {src/internal => include}/pernix/arm64/neon/common.h (100%) rename {src/internal => include}/pernix/arm64/neon/compression.h (100%) rename {src/internal => include}/pernix/arm64/neon/decompression.h (100%) rename {src/internal => include}/pernix/arm64/neon/packing.h (100%) rename {src/internal => include}/pernix/arm64/neon/tables.h (100%) rename {src/internal => include}/pernix/arm64/neon/unpacking.h (100%) rename {src/internal => include}/pernix/arm64/sve2/compression.h (100%) rename {src/internal => include}/pernix/arm64/sve2/decompression.h (100%) rename {src/internal => include}/pernix/arm64/sve2/packing.h (100%) rename {src/internal => include}/pernix/arm64/sve2/tables.h (100%) rename {src/internal => include}/pernix/arm64/sve2/unpacking.h (100%) create mode 100644 include/pernix/backend.hpp create mode 100644 include/pernix/detail/api.hpp create mode 100644 include/pernix/dispatch/cpu_features.h rename {src/internal => include}/pernix/dispatch/kernel.h (100%) rename {src/internal => include}/pernix/dispatch/select.h (64%) create mode 100644 include/pernix/dispatch/select_impl.h rename src/internal/pernix/fallback/avx2_compression.h => include/pernix/fallback/scalar_compression.h (64%) rename src/internal/pernix/fallback/avx2_decompression.h => include/pernix/fallback/scalar_decompression.h (62%) create mode 100644 include/pernix/fallback/simd_compression.h create mode 100644 include/pernix/fallback/simd_decompression.h create mode 100644 include/pernix/fallback/stdpar_compression.h create mode 100644 include/pernix/fallback/stdpar_decompression.h rename {src/internal => include}/pernix/simd_compat.h (100%) rename {src/internal => include}/pernix/x86/avx2/avx2_compression.h (99%) rename {src/internal => include}/pernix/x86/avx2/avx2_decompression.h (99%) rename {src/internal => include}/pernix/x86/avx2/avx2_tables.h (100%) rename {src/internal => include}/pernix/x86/avx512vbmi/avx512vbmi_compression.h (100%) rename {src/internal => include}/pernix/x86/avx512vbmi/avx512vbmi_decompression.h (100%) rename {src/internal => include}/pernix/x86/avx512vbmi/compat.h (100%) rename {src/internal => include}/pernix/x86/avx512vbmi/packing.h (100%) rename {src/internal => include}/pernix/x86/avx512vbmi/tables.h (100%) rename {src/internal => include}/pernix/x86/avx512vbmi/unpacking.h (100%) rename {src/internal => include}/pernix/x86/bmi2/bmi2_compression.h (99%) rename {src/internal => include}/pernix/x86/bmi2/bmi2_decompression.h (100%) rename {src/internal => include}/pernix/x86/utils.h (100%) create mode 100644 src/fallback/simd_compression.cpp create mode 100644 src/fallback/simd_decompression.cpp create mode 100644 src/fallback/stdpar_compression.cpp create mode 100644 src/fallback/stdpar_decompression.cpp delete mode 100644 src/internal/pernix/dispatch/cpu_features.h create mode 100644 tests/header_only_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c6af942..899aae3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,9 @@ option(PERNIX_ENABLE_X86_AVX2 "Build x86 AVX2 backend" ON) option(PERNIX_ENABLE_X86_AVX512VBMI "Build x86 AVX512-VBMI backend" ON) option(PERNIX_ENABLE_ARM64_NEON "Build arm64 NEON backend" ON) option(PERNIX_ENABLE_ARM64_SVE2 "Build arm64 SVE2 backend" ON) +option(PERNIX_ENABLE_FALLBACK_STDPAR "Build fallback stdpar skeleton backend" OFF) +option(PERNIX_ENABLE_FALLBACK_SIMD "Build fallback simd skeleton backend" OFF) +option(PERNIX_ENABLE_FALLBACK_STDPAR_NVCC "Enable NVCC-oriented stdpar skeleton wiring" OFF) option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" off) diff --git a/src/internal/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h similarity index 100% rename from src/internal/pernix/arm64/neon/common.h rename to include/pernix/arm64/neon/common.h diff --git a/src/internal/pernix/arm64/neon/compression.h b/include/pernix/arm64/neon/compression.h similarity index 100% rename from src/internal/pernix/arm64/neon/compression.h rename to include/pernix/arm64/neon/compression.h diff --git a/src/internal/pernix/arm64/neon/decompression.h b/include/pernix/arm64/neon/decompression.h similarity index 100% rename from src/internal/pernix/arm64/neon/decompression.h rename to include/pernix/arm64/neon/decompression.h diff --git a/src/internal/pernix/arm64/neon/packing.h b/include/pernix/arm64/neon/packing.h similarity index 100% rename from src/internal/pernix/arm64/neon/packing.h rename to include/pernix/arm64/neon/packing.h diff --git a/src/internal/pernix/arm64/neon/tables.h b/include/pernix/arm64/neon/tables.h similarity index 100% rename from src/internal/pernix/arm64/neon/tables.h rename to include/pernix/arm64/neon/tables.h diff --git a/src/internal/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h similarity index 100% rename from src/internal/pernix/arm64/neon/unpacking.h rename to include/pernix/arm64/neon/unpacking.h diff --git a/src/internal/pernix/arm64/sve2/compression.h b/include/pernix/arm64/sve2/compression.h similarity index 100% rename from src/internal/pernix/arm64/sve2/compression.h rename to include/pernix/arm64/sve2/compression.h diff --git a/src/internal/pernix/arm64/sve2/decompression.h b/include/pernix/arm64/sve2/decompression.h similarity index 100% rename from src/internal/pernix/arm64/sve2/decompression.h rename to include/pernix/arm64/sve2/decompression.h diff --git a/src/internal/pernix/arm64/sve2/packing.h b/include/pernix/arm64/sve2/packing.h similarity index 100% rename from src/internal/pernix/arm64/sve2/packing.h rename to include/pernix/arm64/sve2/packing.h diff --git a/src/internal/pernix/arm64/sve2/tables.h b/include/pernix/arm64/sve2/tables.h similarity index 100% rename from src/internal/pernix/arm64/sve2/tables.h rename to include/pernix/arm64/sve2/tables.h diff --git a/src/internal/pernix/arm64/sve2/unpacking.h b/include/pernix/arm64/sve2/unpacking.h similarity index 100% rename from src/internal/pernix/arm64/sve2/unpacking.h rename to include/pernix/arm64/sve2/unpacking.h diff --git a/include/pernix/backend.hpp b/include/pernix/backend.hpp new file mode 100644 index 0000000..aa1ea32 --- /dev/null +++ b/include/pernix/backend.hpp @@ -0,0 +1,21 @@ +#ifndef PERNIX_BACKEND_HPP +#define PERNIX_BACKEND_HPP + +#include + +namespace pernix { +enum class Backend { + Auto = PERNIX_BACKEND_AUTO, + Fallback = PERNIX_BACKEND_FALLBACK, + FallbackScalar = PERNIX_BACKEND_FALLBACK_SCALAR, + X86Avx2 = PERNIX_BACKEND_X86_AVX2, + X86Bmi2 = PERNIX_BACKEND_X86_BMI2, + X86Avx512Vbmi = PERNIX_BACKEND_X86_AVX512_VBMI, + Arm64Neon = PERNIX_BACKEND_ARM64_NEON, + Arm64Sve = PERNIX_BACKEND_ARM64_SVE, + FallbackStdpar = PERNIX_BACKEND_FALLBACK_STDPAR, + FallbackSimd = PERNIX_BACKEND_FALLBACK_SIMD +}; +} + +#endif //PERNIX_BACKEND_HPP diff --git a/include/pernix/detail/api.hpp b/include/pernix/detail/api.hpp new file mode 100644 index 0000000..0a4df88 --- /dev/null +++ b/include/pernix/detail/api.hpp @@ -0,0 +1,124 @@ +#ifndef PERNIX_DETAIL_API_HPP +#define PERNIX_DETAIL_API_HPP + +#include +#include + +#include +#include + +namespace pernix::detail { +constexpr bool is_valid_block_size(const u32 block_size) { + return block_size == 64 || block_size == 128 || block_size == 256 || block_size == 512 || block_size == 1024; +} + +inline pernix_status select_error_status(const std::string_view kernel_name) { + if (kernel_name == "invalid_backend") { + return PERNIX_STATUS_UNSUPPORTED_BACKEND; + } + if (kernel_name == "unsupported_implementation") { + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; + } + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; +} + +template +inline pernix_status compress_block(const Backend backend, const u8 bit_width, const u32 block_size, + const void *input, const ScaleType scale, void *output) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + + const auto kernel = [&] { + if constexpr (std::is_same_v) { + return internal::select_compress_block_f32(backend, bit_width, block_size); + } else { + return internal::select_compress_block_f64(backend, bit_width, block_size); + } + }(); + if (!kernel) { + return select_error_status(kernel.name); + } + + return static_cast(kernel.func(input, scale, output)); +} + +template +inline pernix_status compress_blocks(const Backend backend, const u8 bit_width, const u32 block_size, + const void *input, const ScaleType scale, void *output, const u32 blocks) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + + const auto kernel = [&] { + if constexpr (std::is_same_v) { + return internal::select_compress_blocks_f32(backend, bit_width, block_size); + } else { + return internal::select_compress_blocks_f64(backend, bit_width, block_size); + } + }(); + if (!kernel) { + return select_error_status(kernel.name); + } + + return static_cast(kernel.func(input, scale, output, blocks)); +} + +template +inline pernix_status decompress_block(const Backend backend, const u8 bit_width, const u32 block_size, + const void *input, const ScaleType scale, void *output, + const bool sign_values) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + + const auto kernel = [&] { + if constexpr (std::is_same_v) { + return internal::select_decompress_block_f32(backend, bit_width, block_size, sign_values); + } else { + return internal::select_decompress_block_f64(backend, bit_width, block_size, sign_values); + } + }(); + if (!kernel) { + return select_error_status(kernel.name); + } + + return static_cast(kernel.func(input, scale, output)); +} + +template +inline pernix_status decompress_blocks(const Backend backend, const u8 bit_width, const u32 block_size, + const void *input, const ScaleType scale, void *output, const u32 blocks, + const bool sign_values) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + + const auto kernel = [&] { + if constexpr (std::is_same_v) { + return internal::select_decompress_blocks_f32(backend, bit_width, block_size, sign_values); + } else { + return internal::select_decompress_blocks_f64(backend, bit_width, block_size, sign_values); + } + }(); + if (!kernel) { + return select_error_status(kernel.name); + } + + return static_cast(kernel.func(input, scale, output, blocks)); +} +} + +#endif //PERNIX_DETAIL_API_HPP diff --git a/include/pernix/dispatch/cpu_features.h b/include/pernix/dispatch/cpu_features.h new file mode 100644 index 0000000..fc7354e --- /dev/null +++ b/include/pernix/dispatch/cpu_features.h @@ -0,0 +1,113 @@ +#ifndef PERNIX_CPU_FEATURES_H +#define PERNIX_CPU_FEATURES_H + +#include + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386) || defined(_M_IX86) +#if defined(_MSC_VER) +#include +#else +#include +#include +#endif +#endif + +#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) +#include +#ifndef HWCAP_SVE +#define HWCAP_SVE (1 << 22) +#endif +#ifndef HWCAP2_SVE2 +#define HWCAP2_SVE2 (1 << 1) +#endif +#endif + +namespace pernix::internal { +struct CpuFeatures { + bool avx2 = false; + bool bmi2 = false; + bool avx512f = false; + bool avx512dq = false; + bool avx512bw = false; + bool avx512vl = false; + bool avx512vbmi = false; + bool neon = false; + bool sve = false; + bool sve2 = false; +}; + +inline CpuFeatures detect_cpu_features() { + CpuFeatures features{}; + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386) || defined(_M_IX86) + int regs[4]{}; + +#if defined(_MSC_VER) + __cpuidex(regs, 1, 0); + const auto xgetbv = [](const unsigned int index) -> u64 { + return _xgetbv(index); + }; +#else + __cpuid_count(1, 0, regs[0], regs[1], regs[2], regs[3]); + const auto xgetbv = [](const unsigned int index) -> u64 { + u32 eax = 0; + u32 edx = 0; + __asm__ volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(index)); + return static_cast(eax) | (static_cast(edx) << 32U); + }; +#endif + + const bool osxsave = (regs[2] & (1 << 27)) != 0; + const bool avx = (regs[2] & (1 << 28)) != 0; + if (!osxsave || !avx) { + return features; + } + + const u64 xcr0 = xgetbv(0); + const bool xmm_enabled = (xcr0 & 0x2) != 0; + const bool ymm_enabled = (xcr0 & 0x4) != 0; + const bool zmm_enabled = (xcr0 & 0x20) != 0 && (xcr0 & 0x40) != 0 && (xcr0 & 0x80) != 0; + if (!xmm_enabled || !ymm_enabled) { + return features; + } + +#if defined(_MSC_VER) + __cpuidex(regs, 7, 0); +#else + __cpuid_count(7, 0, regs[0], regs[1], regs[2], regs[3]); +#endif + + features.avx2 = (regs[1] & (1 << 5)) != 0; + features.bmi2 = (regs[1] & (1 << 8)) != 0; + if (zmm_enabled) { + features.avx512f = (regs[1] & (1 << 16)) != 0; + features.avx512dq = (regs[1] & (1 << 29)) != 0; + features.avx512bw = (regs[1] & (1 << 30)) != 0; + features.avx512vl = (regs[1] & (1 << 31)) != 0; + features.avx512vbmi = (regs[2] & (1 << 1)) != 0; + } +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) + features.neon = true; +#elif defined(__ARM_NEON) || defined(__ARM_NEON__) + features.neon = true; +#endif + +#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) + const unsigned long hwcap = getauxval(AT_HWCAP); + const unsigned long hwcap2 = getauxval(AT_HWCAP2); + features.sve = (hwcap & HWCAP_SVE) != 0; + features.sve2 = (hwcap2 & HWCAP2_SVE2) != 0; +#endif + + return features; +} + +inline CpuFeatures get_cached_cpu_features() { + static const CpuFeatures features = detect_cpu_features(); + return features; +} +} + +#endif //PERNIX_CPU_FEATURES_H diff --git a/src/internal/pernix/dispatch/kernel.h b/include/pernix/dispatch/kernel.h similarity index 100% rename from src/internal/pernix/dispatch/kernel.h rename to include/pernix/dispatch/kernel.h diff --git a/src/internal/pernix/dispatch/select.h b/include/pernix/dispatch/select.h similarity index 64% rename from src/internal/pernix/dispatch/select.h rename to include/pernix/dispatch/select.h index d00357c..b9deef0 100644 --- a/src/internal/pernix/dispatch/select.h +++ b/include/pernix/dispatch/select.h @@ -1,9 +1,39 @@ #ifndef PERNIX_SELECT_H #define PERNIX_SELECT_H -#include +#include #include +#if !defined(PERNIX_BUILD_X86_AVX2) && !defined(PERNIX_DISABLE_AVX2) +#if defined(PERNIX_USE_SIMDE) || defined(__AVX2__) +#define PERNIX_BUILD_X86_AVX2 1 +#endif +#endif + +#if !defined(PERNIX_BUILD_X86_BMI2) && !defined(PERNIX_DISABLE_BMI2) +#if defined(PERNIX_USE_SIMDE) || (defined(__AVX2__) && defined(__BMI2__)) +#define PERNIX_BUILD_X86_BMI2 1 +#endif +#endif + +#if !defined(PERNIX_BUILD_X86_AVX512_VBMI) && !defined(PERNIX_DISABLE_AVX512) +#if defined(PERNIX_USE_SIMDE) || (defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && defined(__AVX512VL__) && defined(__AVX512VBMI__)) +#define PERNIX_BUILD_X86_AVX512_VBMI 1 +#endif +#endif + +#if !defined(PERNIX_BUILD_ARM64_NEON) +#if defined(__aarch64__) || defined(_M_ARM64) || defined(__ARM_NEON) || defined(__ARM_NEON__) +#define PERNIX_BUILD_ARM64_NEON 1 +#endif +#endif + +#if !defined(PERNIX_BUILD_ARM64_SVE2) +#if defined(__ARM_FEATURE_SVE2) || defined(PERNIX_USE_SIMDE) +#define PERNIX_BUILD_ARM64_SVE2 1 +#endif +#endif + namespace pernix::internal { Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size); @@ -63,6 +93,66 @@ namespace pernix::internal { Kernel select_fallback_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + Kernel select_fallback_scalar_compress_block_f32(u8 bit_width, u32 block_size); + + Kernel select_fallback_scalar_compress_blocks_f32(u8 bit_width, u32 block_size); + + Kernel select_fallback_scalar_compress_block_f64(u8 bit_width, u32 block_size); + + Kernel select_fallback_scalar_compress_blocks_f64(u8 bit_width, u32 block_size); + + Kernel + select_fallback_scalar_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + + Kernel select_fallback_scalar_decompress_blocks_f32(u8 bit_width, u32 block_size, + bool sign_values); + + Kernel + select_fallback_scalar_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + + Kernel select_fallback_scalar_decompress_blocks_f64(u8 bit_width, u32 block_size, + bool sign_values); + + Kernel select_fallback_stdpar_compress_block_f32(u8 bit_width, u32 block_size); + + Kernel select_fallback_stdpar_compress_blocks_f32(u8 bit_width, u32 block_size); + + Kernel select_fallback_stdpar_compress_block_f64(u8 bit_width, u32 block_size); + + Kernel select_fallback_stdpar_compress_blocks_f64(u8 bit_width, u32 block_size); + + Kernel + select_fallback_stdpar_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + + Kernel select_fallback_stdpar_decompress_blocks_f32(u8 bit_width, u32 block_size, + bool sign_values); + + Kernel + select_fallback_stdpar_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + + Kernel select_fallback_stdpar_decompress_blocks_f64(u8 bit_width, u32 block_size, + bool sign_values); + + Kernel select_fallback_simd_compress_block_f32(u8 bit_width, u32 block_size); + + Kernel select_fallback_simd_compress_blocks_f32(u8 bit_width, u32 block_size); + + Kernel select_fallback_simd_compress_block_f64(u8 bit_width, u32 block_size); + + Kernel select_fallback_simd_compress_blocks_f64(u8 bit_width, u32 block_size); + + Kernel + select_fallback_simd_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + + Kernel select_fallback_simd_decompress_blocks_f32(u8 bit_width, u32 block_size, + bool sign_values); + + Kernel + select_fallback_simd_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + + Kernel select_fallback_simd_decompress_blocks_f64(u8 bit_width, u32 block_size, + bool sign_values); + #if defined(PERNIX_BUILD_X86_AVX2) Kernel select_avx2_compress_block_f32(u8 bit_width, u32 block_size); @@ -168,4 +258,6 @@ namespace pernix::internal { #endif } +#include + #endif //PERNIX_SELECT_H diff --git a/include/pernix/dispatch/select_impl.h b/include/pernix/dispatch/select_impl.h new file mode 100644 index 0000000..86aae90 --- /dev/null +++ b/include/pernix/dispatch/select_impl.h @@ -0,0 +1,3210 @@ +#ifndef PERNIX_SELECT_IMPL_H +#define PERNIX_SELECT_IMPL_H + +#include +#include +#include +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +#include +#include +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) +#include +#include +#endif +#if defined(PERNIX_BUILD_X86_AVX2) +#include +#include +#endif +#if defined(PERNIX_BUILD_X86_BMI2) +#include +#include +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) +#include +#include +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) +#include +#include +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) +#include +#include +#endif + + +namespace pernix::internal { + inline bool is_optional_fallback_backend(const Backend backend) { + switch (backend) { + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + default: + return false; + } + } + + inline bool is_compiled_backend(const Backend backend) { + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + return true; +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return true; +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return true; +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return true; +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return true; +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return true; +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return true; +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return true; +#endif + default: + return false; + } + } + + inline bool is_backend_supported_on_machine(const Backend backend) { + const CpuFeatures features = get_cached_cpu_features(); + + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + case Backend::X86Avx2: + return features.avx2; + case Backend::X86Bmi2: + return features.avx2 && features.bmi2; + case Backend::X86Avx512Vbmi: + return features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && + features.avx512vbmi; + case Backend::Arm64Neon: + return features.neon; + case Backend::Arm64Sve: + return features.sve2; + default: + return false; + } + } + + inline Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f32(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } + } + + inline Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f32(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } + } + + inline Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f64(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } + } + + inline Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f64(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } + } + + inline Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, + bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } + } + + inline Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, + bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } + } + + inline Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, + bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } + } + + inline Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, + bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } + } + + inline Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_block_f32(bit_width, block_size); + } + + inline Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_blocks_f32(bit_width, block_size); + } + + inline Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_block_f64(bit_width, block_size); + } + + inline Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_blocks_f64(bit_width, block_size); + } + + inline Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); + } + + inline Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); + } + + inline Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); + } + + inline Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if ( + features.avx512f && + features.avx512dq && + features.avx512bw && + features.avx512vl && + features.avx512vbmi + ) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); + } +} + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ +case N: return Kernel("fallback", &compress_block_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ +case N: return Kernel("fallback", &compress_blocks_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ +case N: return Kernel("fallback", &compress_block_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ +case N: return Kernel("fallback", &compress_blocks_fallback) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: return {"fallback", nullptr}; \ + } + + inline Kernel select_fallback_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } + } + + inline Kernel select_fallback_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } + } + + inline Kernel select_fallback_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } + } + + inline Kernel select_fallback_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } + } + + inline Kernel select_fallback_scalar_compress_block_f32(const u8 bit_width, + const u32 block_size) { + return select_fallback_compress_block_f32(bit_width, block_size); + } + + inline Kernel select_fallback_scalar_compress_blocks_f32(const u8 bit_width, + const u32 block_size) { + return select_fallback_compress_blocks_f32(bit_width, block_size); + } + + inline Kernel select_fallback_scalar_compress_block_f64(const u8 bit_width, + const u32 block_size) { + return select_fallback_compress_block_f64(bit_width, block_size); + } + + inline Kernel select_fallback_scalar_compress_blocks_f64(const u8 bit_width, + const u32 block_size) { + return select_fallback_compress_blocks_f64(bit_width, block_size); + } + + inline Kernel select_fallback_stdpar_compress_block_f32(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + return {"fallback_stdpar", &compress_block_fallback_stdpar<1, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_stdpar_compress_blocks_f32(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + return {"fallback_stdpar", &compress_blocks_fallback_stdpar<1, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_stdpar_compress_block_f64(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + return {"fallback_stdpar", &compress_block_fallback_stdpar<1, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_stdpar_compress_blocks_f64(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + return {"fallback_stdpar", &compress_blocks_fallback_stdpar<1, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_simd_compress_block_f32(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + return {"fallback_simd", &compress_block_fallback_simd<1, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_simd_compress_blocks_f32(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + return {"fallback_simd", &compress_blocks_fallback_simd<1, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_simd_compress_block_f64(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + return {"fallback_simd", &compress_block_fallback_simd<1, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_simd_compress_blocks_f64(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + return {"fallback_simd", &compress_blocks_fallback_simd<1, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) + + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"fallback", nullptr}; \ + } \ + break + + inline Kernel select_fallback_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"fallback", nullptr}; + } + } + + inline Kernel select_fallback_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"fallback", nullptr}; + } + } + + inline Kernel select_fallback_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"fallback", nullptr}; + } + } + + inline Kernel select_fallback_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"fallback", nullptr}; + } + } + + inline Kernel select_fallback_scalar_decompress_block_f32(const u8 bit_width, + const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); + } + + inline Kernel select_fallback_scalar_decompress_blocks_f32(const u8 bit_width, + const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); + } + + inline Kernel select_fallback_scalar_decompress_block_f64(const u8 bit_width, + const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); + } + + inline Kernel select_fallback_scalar_decompress_blocks_f64(const u8 bit_width, + const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); + } + + inline Kernel select_fallback_stdpar_decompress_block_f32(const u8, const u32, + const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + if (sign_values) { + return {"fallback_stdpar", &decompress_block_fallback_stdpar<1, true, 64, f32>}; + } + return {"fallback_stdpar", &decompress_block_fallback_stdpar<1, false, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_stdpar_decompress_blocks_f32(const u8, const u32, + const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + if (sign_values) { + return {"fallback_stdpar", &decompress_blocks_fallback_stdpar<1, true, 64, f32>}; + } + return {"fallback_stdpar", &decompress_blocks_fallback_stdpar<1, false, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_stdpar_decompress_block_f64(const u8, const u32, + const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + if (sign_values) { + return {"fallback_stdpar", &decompress_block_fallback_stdpar<1, true, 64, f64>}; + } + return {"fallback_stdpar", &decompress_block_fallback_stdpar<1, false, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_stdpar_decompress_blocks_f64(const u8, const u32, + const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + if (sign_values) { + return {"fallback_stdpar", &decompress_blocks_fallback_stdpar<1, true, 64, f64>}; + } + return {"fallback_stdpar", &decompress_blocks_fallback_stdpar<1, false, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_simd_decompress_block_f32(const u8, const u32, + const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + if (sign_values) { + return {"fallback_simd", &decompress_block_fallback_simd<1, true, 64, f32>}; + } + return {"fallback_simd", &decompress_block_fallback_simd<1, false, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_simd_decompress_blocks_f32(const u8, const u32, + const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + if (sign_values) { + return {"fallback_simd", &decompress_blocks_fallback_simd<1, true, 64, f32>}; + } + return {"fallback_simd", &decompress_blocks_fallback_simd<1, false, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_simd_decompress_block_f64(const u8, const u32, + const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + if (sign_values) { + return {"fallback_simd", &decompress_block_fallback_simd<1, true, 64, f64>}; + } + return {"fallback_simd", &decompress_block_fallback_simd<1, false, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + + inline Kernel select_fallback_simd_decompress_blocks_f64(const u8, const u32, + const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + if (sign_values) { + return {"fallback_simd", &decompress_blocks_fallback_simd<1, true, 64, f64>}; + } + return {"fallback_simd", &decompress_blocks_fallback_simd<1, false, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif + } + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} +#if defined(PERNIX_BUILD_X86_AVX2) + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ +case N: return Kernel("avx2", &mm256_compress_block_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ +case N: return Kernel("avx2", &mm256_compress_blocks_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ +case N: return Kernel("avx2", &mm256_compress_block_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ +case N: return Kernel("avx2", &mm256_compress_blocks_avx2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: return {"avx2", nullptr}; \ + } + + inline Kernel select_avx2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } + } + + inline Kernel select_avx2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } + } + + inline Kernel select_avx2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } + } + + inline Kernel select_avx2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } + } + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) + + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"avx2", nullptr}; \ + } \ + break + + inline Kernel select_avx2_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"avx2", nullptr}; + } + } + + inline Kernel select_avx2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"avx2", nullptr}; + } + } + + inline Kernel select_avx2_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"avx2", nullptr}; + } + } + + inline Kernel select_avx2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"avx2", nullptr}; + } + } + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ +case N: return Kernel("bmi2", &mm256_compress_block_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ +case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ +case N: return Kernel("bmi2", &mm256_compress_block_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ +case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: return {"bmi2", nullptr}; \ + } + + inline Kernel select_bmi2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } + } + + inline Kernel select_bmi2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } + } + + inline Kernel select_bmi2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } + } + + inline Kernel select_bmi2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } + } + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) + + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"bmi2", nullptr}; \ + } \ + break + + inline Kernel select_bmi2_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"bmi2", nullptr}; + } + } + + inline Kernel select_bmi2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"bmi2", nullptr}; + } + } + + inline Kernel select_bmi2_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"bmi2", nullptr}; + } + } + + inline Kernel select_bmi2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"bmi2", nullptr}; + } + } + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ +case N: return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ +case N: return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ +case N: return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ +case N: return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } + + inline Kernel select_avx512vbmi_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } + } + + inline Kernel select_avx512vbmi_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } + } + + inline Kernel select_avx512vbmi_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } + } + + inline Kernel select_avx512vbmi_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } + } + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) + + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"avx512vbmi", nullptr}; \ + } \ + break + + inline Kernel select_avx512vbmi_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"avx512vbmi", nullptr}; + } + } + + inline Kernel select_avx512vbmi_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"avx512vbmi", nullptr}; + } + } + + inline Kernel select_avx512vbmi_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"avx512vbmi", nullptr}; + } + } + + inline Kernel select_avx512vbmi_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"avx512vbmi", nullptr}; + } + } + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + +namespace pernix::internal { + inline Kernel select_neon_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"neon", nullptr}; + } + + inline Kernel select_neon_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"neon", nullptr}; + } + + inline Kernel select_neon_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"neon", nullptr}; + } + + inline Kernel select_neon_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"neon", nullptr}; + } +} + +using pernix::arm64::neon::neon_decompress_block; +using pernix::arm64::neon::neon_decompress_blocks; + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"neon", nullptr}; \ + } \ + break + +inline Kernel select_neon_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"neon", nullptr}; + } +} + +inline Kernel select_neon_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"neon", nullptr}; + } +} + +inline Kernel select_neon_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"neon", nullptr}; + } +} + +inline Kernel select_neon_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"neon", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + +namespace pernix::internal { + inline Kernel select_sve2_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"sve2", nullptr}; + } + + inline Kernel select_sve2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"sve2", nullptr}; + } + + inline Kernel select_sve2_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"sve2", nullptr}; + } + + inline Kernel select_sve2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void) bit_width; + (void) block_size; + return {"sve2", nullptr}; + } +} + +using pernix::arm64::sve2::sve2_decompress_block; +using pernix::arm64::sve2::sve2_decompress_blocks; + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ +case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ +case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"sve2", nullptr}; \ + } \ + break + + inline Kernel select_sve2_decompress_block_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: return {"sve2", nullptr}; + } + } + + inline Kernel select_sve2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: return {"sve2", nullptr}; + } + } + + inline Kernel select_sve2_decompress_block_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: return {"sve2", nullptr}; + } + } + + inline Kernel select_sve2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: return {"sve2", nullptr}; + } + } + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} +#endif + +#endif //PERNIX_SELECT_IMPL_H diff --git a/src/internal/pernix/fallback/avx2_compression.h b/include/pernix/fallback/scalar_compression.h similarity index 64% rename from src/internal/pernix/fallback/avx2_compression.h rename to include/pernix/fallback/scalar_compression.h index e2c7200..fe8d4f3 100644 --- a/src/internal/pernix/fallback/avx2_compression.h +++ b/include/pernix/fallback/scalar_compression.h @@ -1,5 +1,5 @@ -#ifndef PERNIX_FALLBACK_COMPRESSION_H -#define PERNIX_FALLBACK_COMPRESSION_H +#ifndef PERNIX_FALLBACK_SCALAR_COMPRESSION_H +#define PERNIX_FALLBACK_SCALAR_COMPRESSION_H #include @@ -12,31 +12,14 @@ namespace pernix { namespace internal { - /** - * @brief Quantize a single float value to i32 using the provided scale. - * - * @param input input float value to be quantized. - * @param scale scaling factor used during quantization. - * @return i32 quantized integer value. - */ __always_inline i32 quantize_ps_epi32(const float input, const float scale) { return static_cast(std::lroundf(input * scale)); } - /** - * @brief Quantize a single double value to i64 using the provided scale. - * - * @param input input double value to be quantized. - * @param scale scaling factor used during quantization. - * @return i64 quantized integer value. - */ __always_inline i64 quantize_pd_epi64(const f64 input, const f64 scale) { return std::llround(input * scale); } - /** - * @brief Quantize and clamp without narrowing through an out-of-range integer type. - */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) __always_inline i32 quantize_clamped(const T input, const T scale) { @@ -57,14 +40,10 @@ __always_inline i32 quantize_clamped(const T input, const T scale) { return static_cast(std::llround(scaled)); } - /** - * @brief Clamp a signed quantized value to the representable range of BIT_WIDTH bits. - */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __always_inline i32 clamp_signed_quantized(const i64 value) { if constexpr (BIT_WIDTH == 1) { - // 1-bit fallback is treated as binary quantization (0/1). return static_cast(std::clamp(value, 0, 1)); } @@ -73,16 +52,6 @@ __always_inline i32 clamp_signed_quantized(const i64 value) { return static_cast(std::clamp(value, min_value, max_value)); } - /** - * @brief Append packed scalar values into an output buffer using the selected - * storage width. - * - * @tparam T unsigned integer type used as the packing word. - * @tparam BIT_WIDTH bit width per value in the packed representation. - * @param input vector of quantized values to pack. - * @param bit_offset starting bit offset in the destination buffer. - * @param destination pointer to the output buffer. - */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) void pack_epi32_fallback_inner(const std::vector &input, const u8 bit_offset, @@ -115,15 +84,6 @@ __always_inline i32 clamp_signed_quantized(const i64 value) { } } - /** - * @brief Pack a vector of u32 values into a compact byte representation using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * - * @param input vector of u32 values to be packed. - * @param destination pointer to the output buffer where packed bytes will be stored. - */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) void pack_epi32_fallback(const std::vector &input, u8 * __restrict__ destination) { @@ -135,19 +95,8 @@ __always_inline i32 clamp_signed_quantized(const i64 value) { return internal::pack_epi32_fallback_inner(input, 0, destination); } } - } // namespace internal - - /** - * @brief Compress a single 512-bit block using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - */ + } + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) int compress_block_fallback(const void * __restrict__ input_ptr, const f32 scale, @@ -170,16 +119,6 @@ __always_inline i32 clamp_signed_quantized(const i64 value) { return 0; } - /** - * @brief Compress a single block of double values using the fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) int compress_block_fallback(const void * __restrict__ input_ptr, const f64 scale, @@ -202,18 +141,6 @@ __always_inline i32 clamp_signed_quantized(const i64 value) { return 0; } - /** - * @brief Compress multiple 512-bit blocks using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) int compress_blocks_fallback(const void * __restrict__ input_ptr, float scale, void * __restrict__ output_ptr, @@ -233,17 +160,6 @@ __always_inline i32 clamp_signed_quantized(const i64 value) { return 0; } - /** - * @brief Compress multiple blocks of double values using the fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of blocks to compress. - * @return int status code (0 for success). - */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) int compress_blocks_fallback(const void * __restrict__ input_ptr, const f64 scale, @@ -262,5 +178,6 @@ __always_inline i32 clamp_signed_quantized(const i64 value) { } return 0; } -} // namespace pernix -#endif // PERNIX_FALLBACK_COMPRESSION_H +} + +#endif // PERNIX_FALLBACK_SCALAR_COMPRESSION_H diff --git a/src/internal/pernix/fallback/avx2_decompression.h b/include/pernix/fallback/scalar_decompression.h similarity index 62% rename from src/internal/pernix/fallback/avx2_decompression.h rename to include/pernix/fallback/scalar_decompression.h index 24431e5..720d98a 100644 --- a/src/internal/pernix/fallback/avx2_decompression.h +++ b/include/pernix/fallback/scalar_decompression.h @@ -1,5 +1,5 @@ -#ifndef PERNIX_FALLBACK_DECOMPRESSION_H -#define PERNIX_FALLBACK_DECOMPRESSION_H +#ifndef PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H +#define PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H #include @@ -10,35 +10,14 @@ namespace pernix { namespace internal { - /** -* @brief Dequantize a single i32 value to float using the provided scale. -* -* @param input input i32 value to be dequantized. -* @param scale scaling factor used during quantization. -* @return float dequantized float value. -*/ __always_inline float dequantize_epi32(const i32 input, const float scale) { return static_cast(input) * scale; } - /** -* @brief Dequantize a single i64 value to double using the provided scale. -* -* @param input input i64 value to be dequantized. -* @param scale scaling factor used during quantization. -* @return f64 dequantized double value. -*/ __always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { return static_cast(input) * scale; } - /** -* @brief Sign-extend a packed integer value stored in the low bits of a 32-bit word. -* -* @tparam BIT_WIDTH number of significant bits in the encoded value. -* @param value unsigned packed value. -* @return i32 sign-extended value. -*/ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __always_inline auto sign_extend(const u32 value) -> i32 { @@ -52,17 +31,6 @@ __always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); } - /** -* @brief Unpack bit-packed values from a typed input span into signed 32-bit integers. -* -* @tparam T unsigned integer type used to read the source buffer. -* @tparam BIT_WIDTH bit width per packed value. -* @tparam SIGN_VALUES whether to sign-extend unpacked values. -* @param input pointer to the typed packed input buffer. -* @param bit_offset starting bit offset in the first input word. -* @param elements number of values to unpack. -* @return std::vector unpacked values. -*/ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) __always_inline auto unpack_epi32_fallback_inner(const u8 * __restrict__ input, const u8 bit_offset, @@ -101,15 +69,6 @@ __always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { return output; } - /** -* @brief Unpack packed i32 values from the input buffer using fallback scalar implementation. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @param input pointer to the start of the packed data. -* @param elements number of elements to unpack. -* @return std::vector unpacked i32 values. -*/ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __always_inline auto unpack_epi32_fallback(const u8 * __restrict__ input, @@ -122,18 +81,8 @@ __always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { return unpack_epi32_fallback_inner(input, 0, elements); } } - } // namespace internal - - /** -* @brief Decompress a single 512\-bit block using fallback scalar implementation. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @param input pointer to the start of the compressed block. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed float values will be stored. -* @return int status code (0 for success). -*/ + } + template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) int decompress_block_fallback(const void * __restrict__ input_ptr, const f32 scale, @@ -154,16 +103,6 @@ __always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { return 0; } - /** -* @brief Decompress a single block to double values using the fallback scalar implementation. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @param input pointer to the start of the compressed block. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed double values will be stored. -* @return int status code (0 for success). -*/ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) int decompress_block_fallback(const void * __restrict__ input_ptr, const f64 scale, @@ -184,17 +123,6 @@ __always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { return 0; } - /** -* @brief Decompress multiple 512\-bit blocks using fallback scalar implementation. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @param input pointer to the start of the compressed data. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed float values will be stored. -* @param blocks number of 512-bit blocks to decompress. -* @return int status code (0 for success). -*/ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) int decompress_blocks_fallback(const void * __restrict__ input_ptr, const f32 scale, @@ -214,17 +142,6 @@ __always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { return 0; } - /** -* @brief Decompress multiple blocks to double values using the fallback scalar implementation. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @param input pointer to the start of the compressed data. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed double values will be stored. -* @param blocks number of blocks to decompress. -* @return int status code (0 for success). -*/ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) int decompress_blocks_fallback(const void * __restrict__ input_ptr, const f64 scale, @@ -243,6 +160,6 @@ __always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { return 0; } -} // namespace pernix +} -#endif // PERNIX_FALLBACK_DECOMPRESSION_H +#endif // PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H diff --git a/include/pernix/fallback/simd_compression.h b/include/pernix/fallback/simd_compression.h new file mode 100644 index 0000000..0658cbd --- /dev/null +++ b/include/pernix/fallback/simd_compression.h @@ -0,0 +1,47 @@ +#ifndef PERNIX_FALLBACK_SIMD_COMPRESSION_H +#define PERNIX_FALLBACK_SIMD_COMPRESSION_H + +#include + +#include +#include +#include + +#if defined(__has_include) +#if __has_include() +#include +#endif +#endif + +namespace pernix { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int compress_block_fallback_simd(const void *input_ptr, ScaleType scale, void *output_ptr) { +#if defined(__cpp_lib_simd) + using simd_placeholder = std::simd; + [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); +#endif + [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(elements_per_block); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int compress_blocks_fallback_simd(const void *input_ptr, ScaleType scale, void *output_ptr, u32 blocks) { +#if defined(__cpp_lib_simd) + using simd_placeholder = std::simd; + [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); +#endif + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(blocks); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} +} + +#endif // PERNIX_FALLBACK_SIMD_COMPRESSION_H diff --git a/include/pernix/fallback/simd_decompression.h b/include/pernix/fallback/simd_decompression.h new file mode 100644 index 0000000..4ec8f1c --- /dev/null +++ b/include/pernix/fallback/simd_decompression.h @@ -0,0 +1,51 @@ +#ifndef PERNIX_FALLBACK_SIMD_DECOMPRESSION_H +#define PERNIX_FALLBACK_SIMD_DECOMPRESSION_H + +#include + +#include +#include +#include + +#if defined(__has_include) +#if __has_include() +#include +#endif +#endif + +namespace pernix { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int decompress_block_fallback_simd(const void *input_ptr, ScaleType scale, void *output_ptr) { +#if defined(__cpp_lib_simd) + using simd_placeholder = std::simd; + [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); +#endif + [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; + [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(sign_values); + static_cast(elements_per_block); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int decompress_blocks_fallback_simd(const void *input_ptr, ScaleType scale, void *output_ptr, u32 blocks) { +#if defined(__cpp_lib_simd) + using simd_placeholder = std::simd; + [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); +#endif + [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(blocks); + static_cast(sign_values); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} +} + +#endif // PERNIX_FALLBACK_SIMD_DECOMPRESSION_H diff --git a/include/pernix/fallback/stdpar_compression.h b/include/pernix/fallback/stdpar_compression.h new file mode 100644 index 0000000..79ff59c --- /dev/null +++ b/include/pernix/fallback/stdpar_compression.h @@ -0,0 +1,40 @@ +#ifndef PERNIX_FALLBACK_STDPAR_COMPRESSION_H +#define PERNIX_FALLBACK_STDPAR_COMPRESSION_H + +#include + +#include +#include + +namespace pernix { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int compress_block_fallback_stdpar(const void *input_ptr, ScaleType scale, void *output_ptr) { + [[maybe_unused]] const auto policy = std::execution::par_unseq; + [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; +#if defined(PERNIX_FALLBACK_STDPAR_USE_NVCC) + [[maybe_unused]] constexpr bool nvcc_execution_path = true; +#endif + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(elements_per_block); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int compress_blocks_fallback_stdpar(const void *input_ptr, ScaleType scale, void *output_ptr, u32 blocks) { + [[maybe_unused]] const auto policy = std::execution::par_unseq; +#if defined(PERNIX_FALLBACK_STDPAR_USE_NVCC) + [[maybe_unused]] constexpr bool nvcc_execution_path = true; +#endif + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(blocks); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} +} + +#endif // PERNIX_FALLBACK_STDPAR_COMPRESSION_H diff --git a/include/pernix/fallback/stdpar_decompression.h b/include/pernix/fallback/stdpar_decompression.h new file mode 100644 index 0000000..720ded9 --- /dev/null +++ b/include/pernix/fallback/stdpar_decompression.h @@ -0,0 +1,44 @@ +#ifndef PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H +#define PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H + +#include + +#include +#include + +namespace pernix { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int decompress_block_fallback_stdpar(const void *input_ptr, ScaleType scale, void *output_ptr) { + [[maybe_unused]] const auto policy = std::execution::par_unseq; + [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; + [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; +#if defined(PERNIX_FALLBACK_STDPAR_USE_NVCC) + [[maybe_unused]] constexpr bool nvcc_execution_path = true; +#endif + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(sign_values); + static_cast(elements_per_block); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int decompress_blocks_fallback_stdpar(const void *input_ptr, ScaleType scale, void *output_ptr, u32 blocks) { + [[maybe_unused]] const auto policy = std::execution::par_unseq; + [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; +#if defined(PERNIX_FALLBACK_STDPAR_USE_NVCC) + [[maybe_unused]] constexpr bool nvcc_execution_path = true; +#endif + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(blocks); + static_cast(sign_values); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} +} + +#endif // PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H diff --git a/include/pernix/pernix.h b/include/pernix/pernix.h index 74d586b..5077a14 100644 --- a/include/pernix/pernix.h +++ b/include/pernix/pernix.h @@ -12,17 +12,22 @@ typedef enum pernix_status { PERNIX_STATUS_INVALID_ARGUMENT = -1, PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH = -2, PERNIX_STATUS_UNSUPPORTED_BACKEND = -3, - PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE = -4 + PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE = -4, + PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION = -5, + PERNIX_STATUS_UNSUPPORTED_IMPLEMENTAION = PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION } pernix_status; typedef enum pernix_backend { PERNIX_BACKEND_AUTO = 0, PERNIX_BACKEND_FALLBACK = 1, + PERNIX_BACKEND_FALLBACK_SCALAR = PERNIX_BACKEND_FALLBACK, PERNIX_BACKEND_X86_AVX2 = 2, PERNIX_BACKEND_X86_BMI2 = 3, PERNIX_BACKEND_X86_AVX512_VBMI = 4, PERNIX_BACKEND_ARM64_NEON = 5, - PERNIX_BACKEND_ARM64_SVE = 6 + PERNIX_BACKEND_ARM64_SVE = 6, + PERNIX_BACKEND_FALLBACK_STDPAR = 7, + PERNIX_BACKEND_FALLBACK_SIMD = 8 } pernix_backend; PERNIX_API pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, diff --git a/include/pernix/pernix.hpp b/include/pernix/pernix.hpp index d197f72..d0e7ac4 100644 --- a/include/pernix/pernix.hpp +++ b/include/pernix/pernix.hpp @@ -1,78 +1,58 @@ #ifndef PERNIX_HPP #define PERNIX_HPP +#include +#include + #include -#include namespace pernix { -enum class Backend { - Auto = PERNIX_BACKEND_AUTO, - Fallback = PERNIX_BACKEND_FALLBACK, - X86Avx2 = PERNIX_BACKEND_X86_AVX2, - X86Bmi2 = PERNIX_BACKEND_X86_BMI2, - X86Avx512Vbmi = PERNIX_BACKEND_X86_AVX512_VBMI, - Arm64Neon = PERNIX_BACKEND_ARM64_NEON, - Arm64Sve = PERNIX_BACKEND_ARM64_SVE -}; - __always_inline int compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const float scale, std::span output) { - return pernix_compress_block_f32(static_cast(backend), bit_width, block_size, input.data(), - scale, output.data()); + return detail::compress_block(backend, bit_width, block_size, input.data(), scale, output.data()); } __always_inline int compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const double scale, std::span output) { - return pernix_compress_block_f64(static_cast(backend), bit_width, block_size, input.data(), - scale, output.data()); + return detail::compress_block(backend, bit_width, block_size, input.data(), scale, output.data()); } __always_inline int decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const float scale, std::span output, const bool sign_values = true) { - return pernix_decompress_block_f32(static_cast(backend), bit_width, block_size, input.data(), - scale, output.data(), - sign_values); + return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } __always_inline int decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const double scale, std::span output, const bool sign_values = true) { - return pernix_decompress_block_f64(static_cast(backend), bit_width, block_size, input.data(), - scale, output.data(), - sign_values); + return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } __always_inline int compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const float scale, std::span output, const u32 blocks) { - return pernix_compress_blocks_f32(static_cast(backend), bit_width, block_size, input.data(), - scale, output.data(), - blocks); + return detail::compress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks); } __always_inline int compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const double scale, std::span output, const u32 blocks) { - return pernix_compress_blocks_f64(static_cast(backend), bit_width, block_size, input.data(), - scale, output.data(), - blocks); + return detail::compress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks); } __always_inline int decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const float scale, std::span output, const u32 blocks, const bool sign_values = true) { - return pernix_decompress_blocks_f32(static_cast(backend), bit_width, block_size, input.data(), - scale, output.data(), - blocks, sign_values); + return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, + sign_values); } __always_inline int decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const double scale, std::span output, const u32 blocks, const bool sign_values = true) { - return pernix_decompress_blocks_f64(static_cast(backend), bit_width, block_size, input.data(), - scale, output.data(), - blocks, sign_values); + return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, + sign_values); } // convenience overloads without backend (defaults to Auto) @@ -161,6 +141,6 @@ __always_inline int decompress_blocks(const u8 bit_width, const std::span -#include +#include #include #include diff --git a/src/internal/pernix/x86/avx2/avx2_decompression.h b/include/pernix/x86/avx2/avx2_decompression.h similarity index 99% rename from src/internal/pernix/x86/avx2/avx2_decompression.h rename to include/pernix/x86/avx2/avx2_decompression.h index 2f50018..93f2edd 100644 --- a/src/internal/pernix/x86/avx2/avx2_decompression.h +++ b/include/pernix/x86/avx2/avx2_decompression.h @@ -2,7 +2,7 @@ #define PERNIX_AVX2_DECOMPRESSION_H #include -#include +#include #include #include diff --git a/src/internal/pernix/x86/avx2/avx2_tables.h b/include/pernix/x86/avx2/avx2_tables.h similarity index 100% rename from src/internal/pernix/x86/avx2/avx2_tables.h rename to include/pernix/x86/avx2/avx2_tables.h diff --git a/src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h similarity index 100% rename from src/internal/pernix/x86/avx512vbmi/avx512vbmi_compression.h rename to include/pernix/x86/avx512vbmi/avx512vbmi_compression.h diff --git a/src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h similarity index 100% rename from src/internal/pernix/x86/avx512vbmi/avx512vbmi_decompression.h rename to include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h diff --git a/src/internal/pernix/x86/avx512vbmi/compat.h b/include/pernix/x86/avx512vbmi/compat.h similarity index 100% rename from src/internal/pernix/x86/avx512vbmi/compat.h rename to include/pernix/x86/avx512vbmi/compat.h diff --git a/src/internal/pernix/x86/avx512vbmi/packing.h b/include/pernix/x86/avx512vbmi/packing.h similarity index 100% rename from src/internal/pernix/x86/avx512vbmi/packing.h rename to include/pernix/x86/avx512vbmi/packing.h diff --git a/src/internal/pernix/x86/avx512vbmi/tables.h b/include/pernix/x86/avx512vbmi/tables.h similarity index 100% rename from src/internal/pernix/x86/avx512vbmi/tables.h rename to include/pernix/x86/avx512vbmi/tables.h diff --git a/src/internal/pernix/x86/avx512vbmi/unpacking.h b/include/pernix/x86/avx512vbmi/unpacking.h similarity index 100% rename from src/internal/pernix/x86/avx512vbmi/unpacking.h rename to include/pernix/x86/avx512vbmi/unpacking.h diff --git a/src/internal/pernix/x86/bmi2/bmi2_compression.h b/include/pernix/x86/bmi2/bmi2_compression.h similarity index 99% rename from src/internal/pernix/x86/bmi2/bmi2_compression.h rename to include/pernix/x86/bmi2/bmi2_compression.h index e165c3a..8a0cc57 100644 --- a/src/internal/pernix/x86/bmi2/bmi2_compression.h +++ b/include/pernix/x86/bmi2/bmi2_compression.h @@ -2,7 +2,7 @@ #define PERNIX_BMI2_COMPRESSION_H #include -#include +#include #include #include diff --git a/src/internal/pernix/x86/bmi2/bmi2_decompression.h b/include/pernix/x86/bmi2/bmi2_decompression.h similarity index 100% rename from src/internal/pernix/x86/bmi2/bmi2_decompression.h rename to include/pernix/x86/bmi2/bmi2_decompression.h diff --git a/src/internal/pernix/x86/utils.h b/include/pernix/x86/utils.h similarity index 100% rename from src/internal/pernix/x86/utils.h rename to include/pernix/x86/utils.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index aab5c67..41e21c6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,17 +2,13 @@ include(GNUInstallDirs) include(CMakePackageConfigHelpers) add_library(pernix SHARED) -target_sources(pernix - PRIVATE - pernix.cpp - - fallback/fallback_compression.cpp - fallback/fallback_decompression.cpp - - dispatch/select.cpp -) +add_library(pernix_header_only INTERFACE) +add_library(pernix::pernix_header_only ALIAS pernix_header_only) +target_sources(pernix PRIVATE pernix.cpp) target_compile_features(pernix PUBLIC cxx_std_20) +target_compile_features(pernix_header_only INTERFACE cxx_std_20) +target_compile_options(pernix PRIVATE ${PERNIX_PRIVATE_COMPILE_OPTIONS}) set_target_properties(pernix PROPERTIES OUTPUT_NAME "pernix" @@ -24,260 +20,105 @@ target_include_directories(pernix PUBLIC $ $ - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/internal ) -target_compile_definitions(pernix - PRIVATE - PERNIX_BUILD_LIB=1 +target_include_directories(pernix_header_only + INTERFACE + $ + $ ) +target_compile_definitions(pernix PRIVATE PERNIX_BUILD_LIB=1) + +if (PERNIX_ENABLE_LTO) + set_target_properties(pernix PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) +endif () + if (PERNIX_USE_SIMDE) - # target_link_libraries(pernix PUBLIC simde::simde) target_compile_definitions(pernix PUBLIC PERNIX_USE_SIMDE=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_USE_SIMDE=1) endif () -if (PERNIX_TARGET_IS_X86) - target_sources(pernix - PRIVATE - dispatch/cpu_features_x86.cpp - ) - - if (MSVC) - set_source_files_properties( - dispatch/cpu_features_x86.cpp - PROPERTIES - COMPILE_OPTIONS "/arch:AVX512" - ) - else () - set_source_files_properties( - dispatch/cpu_features_x86.cpp - PROPERTIES - COMPILE_OPTIONS "-mavx;-mbmi;-mbmi2;-mavx2;-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512cd;-mavx512vbmi" - ) - endif () +if (PERNIX_ENABLE_FALLBACK_STDPAR) + target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_STDPAR=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_STDPAR=1) endif () -if (PERNIX_TARGET_IS_ARM64) - target_sources(pernix - PRIVATE - dispatch/cpu_features_arm.cpp - ) +if (PERNIX_ENABLE_FALLBACK_SIMD) + target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_SIMD=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_SIMD=1) +endif () - set_target_properties(pernix PROPERTIES - COMPILE_OPTIONS "-march=armv8-a+simd+sve2" - ) +if (PERNIX_ENABLE_FALLBACK_STDPAR_NVCC) + target_compile_definitions(pernix PUBLIC PERNIX_FALLBACK_STDPAR_USE_NVCC=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_FALLBACK_STDPAR_USE_NVCC=1) endif () if (PERNIX_ENABLE_X86_BMI2 AND PERNIX_TARGET_IS_X86) - target_sources(pernix - PRIVATE - x86/bmi2/bmi2_compression.cpp - x86/bmi2/bmi2_decompression.cpp - ) - - target_compile_definitions(pernix - PRIVATE - PERNIX_BUILD_X86_BMI2=1 - ) - - if (MSVC) - # BMI2 is not exposed with MSVC - else () - set_source_files_properties( - x86/bmi2/bmi2_compression.cpp - x86/bmi2/bmi2_decompression.cpp - PROPERTIES - COMPILE_OPTIONS "-mbmi;-mbmi2;-mavx2" - ) - endif () - + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_BMI2=1) endif () if (PERNIX_ENABLE_X86_AVX2 AND PERNIX_TARGET_IS_X86) - target_sources(pernix - PRIVATE - x86/avx2/avx2_compression.cpp - x86/avx2/avx2_decompression.cpp - ) - - target_compile_definitions(pernix - PRIVATE - PERNIX_BUILD_X86_AVX2=1 - ) - - if (MSVC) - set_source_files_properties( - x86/avx2/avx2_compression.cpp - x86/avx2/avx2_decompression.cpp - PROPERTIES - COMPILE_OPTIONS "/arch:AVX2" - ) - else () - set_source_files_properties( - x86/avx2/avx2_compression.cpp - x86/avx2/avx2_decompression.cpp - PROPERTIES - COMPILE_OPTIONS "-mavx2" - ) - endif () + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_AVX2=1) endif () if (PERNIX_ENABLE_X86_AVX512VBMI AND PERNIX_TARGET_IS_X86) - target_sources(pernix - PRIVATE - x86/avx512vbmi/avx512vbmi_compression.cpp - x86/avx512vbmi/avx512vbmi_decompression.cpp - ) - - target_compile_definitions(pernix - PRIVATE - PERNIX_BUILD_X86_AVX512_VBMI=1 - ) - - if (MSVC) - set_source_files_properties( - x86/avx512vbmi/avx512vbmi_compression.cpp - x86/avx512vbmi/avx512vbmi_decompression.cpp - PROPERTIES - COMPILE_OPTIONS "/arch:AVX512" - ) - else () - set_source_files_properties( - x86/avx512vbmi/avx512vbmi_compression.cpp - x86/avx512vbmi/avx512vbmi_decompression.cpp - PROPERTIES - COMPILE_OPTIONS "-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512cd;-mavx512vbmi" - ) - endif () + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_AVX512_VBMI=1) endif () if (PERNIX_ENABLE_ARM64_NEON AND PERNIX_TARGET_IS_ARM64) - target_sources(pernix - PRIVATE - arm64/neon/compression.cpp - arm64/neon/decompression.cpp - ) - - target_compile_definitions(pernix - PRIVATE - PERNIX_BUILD_ARM64_NEON=1 - ) - - set_source_files_properties( - arm64/neon/compression.cpp - arm64/neon/decompression.cpp - PROPERTIES - COMPILE_OPTIONS "-march=armv8-a+simd" - ) + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_ARM64_NEON=1) endif () if (PERNIX_ENABLE_ARM64_SVE2 AND PERNIX_TARGET_IS_ARM64) - target_sources(pernix - PRIVATE - arm64/sve2/compression.cpp - arm64/sve2/decompression.cpp - ) - - target_compile_definitions(pernix - PRIVATE - PERNIX_BUILD_ARM64_SVE2=1 - ) - - set_source_files_properties( - arm64/sve2/compression.cpp - arm64/sve2/decompression.cpp - PROPERTIES - COMPILE_OPTIONS "-march=armv8.2-a+simd+sve2" - ) + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_ARM64_SVE2=1) endif () -#[===[ -file(GLOB_RECURSE - PERNIX_COMMON_SOURCES - ./fallback/*.cpp - pernix.cpp - dispatch/select.cpp -) - -set(PERNIX_SOURCES ${PERNIX_COMMON_SOURCES}) - -if (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "X86") - file(GLOB_RECURSE - PERNIX_X86_SOURCES - CONFIGURE_DEPENDS - ./x86/*.cpp - ) - list(APPEND PERNIX_SOURCES ${PERNIX_X86_SOURCES}) -elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_NEON") - file(GLOB_RECURSE - PERNIX_ARM64_NEON_SOURCES - CONFIGURE_DEPENDS - ./arm64/neon/*.cpp - ) - list(APPEND PERNIX_SOURCES ${PERNIX_ARM64_NEON_SOURCES}) -elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE") - file(GLOB_RECURSE - PERNIX_ARM64_SVE_SOURCES - CONFIGURE_DEPENDS - ./arm64/sve/*.cpp - ) - list(APPEND PERNIX_SOURCES ${PERNIX_ARM64_SVE_SOURCES}) -elseif (PERNIX_SELECTED_ARCH_BACKEND STREQUAL "ARM64_SVE2") - file(GLOB_RECURSE - PERNIX_ARM64_SVE2_SOURCES - CONFIGURE_DEPENDS - ./arm64/sve2/*.cpp - ) - list(APPEND PERNIX_SOURCES ${PERNIX_ARM64_SVE2_SOURCES}) -endif () - - -add_library(pernix SHARED ${PERNIX_SOURCES}) -add_library(pernix::pernix ALIAS pernix) -set_target_properties(pernix PROPERTIES - OUTPUT_NAME "pernix" - VERSION ${NORMALIZED_VERSION} -) -target_compile_features(pernix PUBLIC cxx_std_20) -target_compile_options(pernix PRIVATE ${PERNIX_PRIVATE_COMPILE_OPTIONS}) -target_include_directories(pernix - PUBLIC - $ - $ - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/internal -) -if (PERNIX_ENABLE_LTO) - set_target_properties(pernix PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) +if (PERNIX_TARGET_IS_X86) + if (MSVC) + if (PERNIX_ENABLE_X86_AVX512VBMI) + target_compile_options(pernix PRIVATE /arch:AVX512) + elseif (PERNIX_ENABLE_X86_AVX2 OR PERNIX_ENABLE_X86_BMI2) + target_compile_options(pernix PRIVATE /arch:AVX2) + endif () + else () + if (PERNIX_ENABLE_X86_AVX512VBMI) + target_compile_options(pernix PRIVATE -mavx -mbmi -mbmi2 -mavx2 -mavx512f -mavx512bw -mavx512vl -mavx512dq -mavx512cd -mavx512vbmi) + elseif (PERNIX_ENABLE_X86_BMI2) + target_compile_options(pernix PRIVATE -mbmi -mbmi2 -mavx2) + elseif (PERNIX_ENABLE_X86_AVX2) + target_compile_options(pernix PRIVATE -mavx2) + endif () + endif () endif () -if (PERNIX_USE_SIMDE) - target_link_libraries(pernix PUBLIC simde::simde) - target_compile_definitions(pernix PUBLIC PERNIX_USE_SIMDE=1) +if (PERNIX_TARGET_IS_ARM64) + if (PERNIX_ENABLE_ARM64_SVE2) + target_compile_options(pernix PRIVATE -march=armv8.2-a+simd+sve2) + elseif (PERNIX_ENABLE_ARM64_NEON) + target_compile_options(pernix PRIVATE -march=armv8-a+simd) + endif () endif () -target_compile_definitions(pernix PUBLIC "PERNIX_BACKEND_${PERNIX_SELECTED_ARCH_BACKEND}=1") - if (PERNIX_DISABLE_BMI2) target_compile_definitions(pernix PUBLIC PERNIX_DISABLE_BMI2=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_DISABLE_BMI2=1) endif () + if (PERNIX_DISABLE_AVX2) target_compile_definitions(pernix PUBLIC PERNIX_DISABLE_AVX2=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_DISABLE_AVX2=1) endif () + if (PERNIX_DISABLE_AVX512) target_compile_definitions(pernix PUBLIC PERNIX_DISABLE_AVX512=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_DISABLE_AVX512=1) endif () -set_target_properties(pernix PROPERTIES LINKER_LANGUAGE CXX) - -configure_file( - "${PROJECT_SOURCE_DIR}/cmake/pernix.pc.in" "${PROJECT_BINARY_DIR}/pernix.pc" @ONLY -) +configure_file("${PROJECT_SOURCE_DIR}/cmake/pernix.pc.in" "${PROJECT_BINARY_DIR}/pernix.pc" @ONLY) if (PERNIX_ENABLE_INSTALL) - install(TARGETS pernix + install(TARGETS pernix pernix_header_only EXPORT pernixTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -288,6 +129,7 @@ if (PERNIX_ENABLE_INSTALL) DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.*h" ) + if (PERNIX_USE_SIMDE AND PERNIX_BUNDLE_SIMDE_FOR_INSTALL) install(DIRECTORY "${simde_SOURCE_DIR}/simde" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -317,11 +159,8 @@ if (PERNIX_ENABLE_INSTALL) NAMESPACE pernix:: DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/pernix" ) - - install( - FILES ${PROJECT_BINARY_DIR}/pernix.pc - DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" - ) + install(FILES "${PROJECT_BINARY_DIR}/pernix.pc" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") endif () if (PERNIX_ENABLE_DOXYGEN) @@ -338,9 +177,7 @@ if (PERNIX_ENABLE_DOXYGEN) set(DOXYGEN_SHOW_INCLUDE_FILES YES) set(DOXYGEN_BINARY_TOC YES) set(DOXYGEN_TOC_EXPAND YES) - # set(DOXYGEN_USE_MDFILE_AS_MAINPAGE "index.md") doxygen_add_docs(pernix_doxygen - # docs include src ALL @@ -352,5 +189,3 @@ if (PERNIX_ENABLE_DOXYGEN) DESTINATION ${CMAKE_INSTALL_DOCDIR}) endif () endif () - -]===] \ No newline at end of file diff --git a/src/dispatch/select.cpp b/src/dispatch/select.cpp index ca4673e..d4e2264 100644 --- a/src/dispatch/select.cpp +++ b/src/dispatch/select.cpp @@ -2,12 +2,100 @@ #include namespace pernix::internal { + bool is_optional_fallback_backend(const Backend backend) { + switch (backend) { + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + default: + return false; + } + } + + bool is_compiled_backend(const Backend backend) { + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + return true; +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return true; +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return true; +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return true; +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return true; +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return true; +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return true; +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return true; +#endif + default: + return false; + } + } + + bool is_backend_supported_on_machine(const Backend backend) { + const CpuFeatures features = get_cached_cpu_features(); + + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + case Backend::X86Avx2: + return features.avx2; + case Backend::X86Bmi2: + return features.avx2 && features.bmi2; + case Backend::X86Avx512Vbmi: + return features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && + features.avx512vbmi; + case Backend::Arm64Neon: + return features.neon; + case Backend::Arm64Sve: + return features.sve2; + default: + return false; + } + } + Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } switch (backend) { case Backend::Auto: return select_auto_compress_block_f32(bit_width, block_size); case Backend::Fallback: return select_fallback_compress_block_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f32(bit_width, block_size); +#endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) case Backend::X86Avx512Vbmi: return select_avx512vbmi_compress_block_f32(bit_width, block_size); @@ -38,11 +126,26 @@ namespace pernix::internal { } Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } switch (backend) { case Backend::Auto: return select_auto_compress_blocks_f32(bit_width, block_size); case Backend::Fallback: return select_fallback_compress_blocks_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f32(bit_width, block_size); +#endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) case Backend::X86Avx512Vbmi: return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); @@ -73,11 +176,26 @@ namespace pernix::internal { } Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } switch (backend) { case Backend::Auto: return select_auto_compress_block_f64(bit_width, block_size); case Backend::Fallback: return select_fallback_compress_block_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f64(bit_width, block_size); +#endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) case Backend::X86Avx512Vbmi: return select_avx512vbmi_compress_block_f64(bit_width, block_size); @@ -108,11 +226,26 @@ namespace pernix::internal { } Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } switch (backend) { case Backend::Auto: return select_auto_compress_blocks_f64(bit_width, block_size); case Backend::Fallback: return select_fallback_compress_blocks_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f64(bit_width, block_size); +#endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) case Backend::X86Avx512Vbmi: return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); @@ -144,11 +277,26 @@ namespace pernix::internal { Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } switch (backend) { case Backend::Auto: return select_auto_decompress_block_f32(bit_width, block_size, sign_values); case Backend::Fallback: return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f32(bit_width, block_size, sign_values); +#endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) case Backend::X86Avx512Vbmi: return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); @@ -180,11 +328,26 @@ namespace pernix::internal { Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } switch (backend) { case Backend::Auto: return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); case Backend::Fallback: return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) case Backend::X86Avx512Vbmi: return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); @@ -216,11 +379,26 @@ namespace pernix::internal { Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } switch (backend) { case Backend::Auto: return select_auto_decompress_block_f64(bit_width, block_size, sign_values); case Backend::Fallback: return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f64(bit_width, block_size, sign_values); +#endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) case Backend::X86Avx512Vbmi: return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); @@ -252,11 +430,26 @@ namespace pernix::internal { Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } switch (backend) { case Backend::Auto: return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); case Backend::Fallback: return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) case Backend::X86Avx512Vbmi: return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); diff --git a/src/fallback/fallback_compression.cpp b/src/fallback/fallback_compression.cpp index 4f1e955..021ac27 100644 --- a/src/fallback/fallback_compression.cpp +++ b/src/fallback/fallback_compression.cpp @@ -1,6 +1,6 @@ #include -#include -#include +#include +#include namespace pernix::internal { #define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ diff --git a/src/fallback/fallback_decompression.cpp b/src/fallback/fallback_decompression.cpp index 184397f..72d5866 100644 --- a/src/fallback/fallback_decompression.cpp +++ b/src/fallback/fallback_decompression.cpp @@ -1,5 +1,5 @@ #include -#include +#include namespace pernix::internal { #define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ diff --git a/src/fallback/simd_compression.cpp b/src/fallback/simd_compression.cpp new file mode 100644 index 0000000..cb9a467 --- /dev/null +++ b/src/fallback/simd_compression.cpp @@ -0,0 +1,4 @@ +#include + +// Header-only selector definitions live in select_impl.h. +// This translation unit reserves the compiled std::simd compression backend slot. diff --git a/src/fallback/simd_decompression.cpp b/src/fallback/simd_decompression.cpp new file mode 100644 index 0000000..fe09cc3 --- /dev/null +++ b/src/fallback/simd_decompression.cpp @@ -0,0 +1,4 @@ +#include + +// Header-only selector definitions live in select_impl.h. +// This translation unit reserves the compiled std::simd decompression backend slot. diff --git a/src/fallback/stdpar_compression.cpp b/src/fallback/stdpar_compression.cpp new file mode 100644 index 0000000..3db39cc --- /dev/null +++ b/src/fallback/stdpar_compression.cpp @@ -0,0 +1,4 @@ +#include + +// Header-only selector definitions live in select_impl.h. +// This translation unit reserves the compiled stdpar compression backend slot. diff --git a/src/fallback/stdpar_decompression.cpp b/src/fallback/stdpar_decompression.cpp new file mode 100644 index 0000000..d0768d7 --- /dev/null +++ b/src/fallback/stdpar_decompression.cpp @@ -0,0 +1,4 @@ +#include + +// Header-only selector definitions live in select_impl.h. +// This translation unit reserves the compiled stdpar decompression backend slot. diff --git a/src/internal/pernix/dispatch/cpu_features.h b/src/internal/pernix/dispatch/cpu_features.h deleted file mode 100644 index bcf3e7d..0000000 --- a/src/internal/pernix/dispatch/cpu_features.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef PERNIX_CPU_FEATURES_H -#define PERNIX_CPU_FEATURES_H - -namespace pernix::internal { -struct CpuFeatures { - bool avx2 = false; - bool bmi2 = false; - bool avx512f = false; - bool avx512dq = false; - bool avx512bw = false; - bool avx512vl = false; - bool avx512vbmi = false; - bool neon = false; - bool sve = false; - bool sve2 = false; -}; - -CpuFeatures detect_cpu_features(); - -inline CpuFeatures get_cached_cpu_features() { - static const CpuFeatures features = detect_cpu_features(); - return features; -} -} - -#endif //PERNIX_CPU_FEATURES_H diff --git a/src/pernix.cpp b/src/pernix.cpp index a57a2e5..db8ab05 100644 --- a/src/pernix.cpp +++ b/src/pernix.cpp @@ -1,173 +1,51 @@ -#include -#include - -namespace { - bool is_valid_block_size(u32 block_size) { - return block_size == 64 || block_size == 128 || block_size == 256 || block_size == 512 || block_size == 1024; - } -} +#include extern "C" { pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, float scale, void *output) { - if (input == nullptr || output == nullptr) { - return PERNIX_STATUS_INVALID_ARGUMENT; - } - if (!is_valid_block_size(block_size)) { - return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; - } - - const auto kernel = pernix::internal::select_compress_block_f32(static_cast(backend), bit_width, - block_size); - - if (!kernel) { - return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; - } - - return static_cast(kernel.func(input, scale, output)); + return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, + output); } pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, float scale, void *output, u32 blocks) { - if (input == nullptr || output == nullptr) { - return PERNIX_STATUS_INVALID_ARGUMENT; - } - - if (!is_valid_block_size(block_size)) { - return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; - } - - const auto kernel = pernix::internal::select_compress_blocks_f32(static_cast(backend), bit_width, - block_size); - - if (!kernel) { - return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; - } - - return static_cast(kernel.func(input, scale, output, blocks)); + return pernix::detail::compress_blocks(static_cast(backend), bit_width, block_size, input, + scale, output, blocks); } pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, float scale, void *output, bool sign_values) { - if (input == nullptr || output == nullptr) { - return PERNIX_STATUS_INVALID_ARGUMENT; - } - - if (!is_valid_block_size(block_size)) { - return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; - } - - const auto kernel = pernix::internal::select_decompress_block_f32(static_cast(backend), bit_width, - block_size, - sign_values); - - if (!kernel) { - return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; - } - - return static_cast(kernel.func(input, scale, output)); + return pernix::detail::decompress_block(static_cast(backend), bit_width, block_size, input, + scale, output, sign_values); } pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, float scale, void *output, u32 blocks, bool sign_values) { - if (input == nullptr || output == nullptr) { - return PERNIX_STATUS_INVALID_ARGUMENT; - } - - if (!is_valid_block_size(block_size)) { - return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; - } - - const auto kernel = pernix::internal::select_decompress_blocks_f32(static_cast(backend), bit_width, - block_size, - sign_values); - - if (!kernel) { - return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; - } - - return static_cast(kernel.func(input, scale, output, blocks)); + return pernix::detail::decompress_blocks(static_cast(backend), bit_width, block_size, input, + scale, output, blocks, sign_values); } pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, double scale, void *output) { - if (input == nullptr || output == nullptr) { - return PERNIX_STATUS_INVALID_ARGUMENT; - } - - if (!is_valid_block_size(block_size)) { - return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; - } - - const auto kernel = pernix::internal::select_compress_block_f64(static_cast(backend), bit_width, - block_size); - - if (!kernel) { - return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; - } - - return static_cast(kernel.func(input, scale, output)); + return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, + output); } pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, double scale, void *output, u32 blocks) { - if (input == nullptr || output == nullptr) { - return PERNIX_STATUS_INVALID_ARGUMENT; - } - - if (!is_valid_block_size(block_size)) { - return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; - } - - const auto kernel = pernix::internal::select_compress_blocks_f64(static_cast(backend), bit_width, - block_size); - - if (!kernel) { - return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; - } - - return static_cast(kernel.func(input, scale, output, blocks)); + return pernix::detail::compress_blocks(static_cast(backend), bit_width, block_size, input, + scale, output, blocks); } pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, double scale, void *output, bool sign_values) { - if (input == nullptr || output == nullptr) { - return PERNIX_STATUS_INVALID_ARGUMENT; - } - - if (!is_valid_block_size(block_size)) { - return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; - } - - const auto kernel = pernix::internal::select_decompress_block_f64(static_cast(backend), bit_width, - block_size, - sign_values); - - if (!kernel) { - return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; - } - - return static_cast(kernel.func(input, scale, output)); + return pernix::detail::decompress_block(static_cast(backend), bit_width, block_size, input, + scale, output, sign_values); } pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, double scale, void *output, u32 blocks, bool sign_values) { - if (input == nullptr || output == nullptr) { - return PERNIX_STATUS_INVALID_ARGUMENT; - } - - if (!is_valid_block_size(block_size)) { - return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; - } - - const auto kernel = pernix::internal::select_decompress_blocks_f64(static_cast(backend), bit_width, - block_size, - sign_values); - - if (!kernel) { - return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; - } - - return static_cast(kernel.func(input, scale, output, blocks)); + return pernix::detail::decompress_blocks(static_cast(backend), bit_width, block_size, input, + scale, output, blocks, sign_values); } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 62bb90e..a2803dc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,6 +2,7 @@ find_package(PkgConfig) pkg_search_module(GTEST REQUIRED gtest) file(GLOB PERNIX_TEST_SOURCE_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) +list(FILTER PERNIX_TEST_SOURCE_FILES EXCLUDE REGEX ".*/header_only_tests\\.cpp$") set(PERNIX_TEST_BLOCK_SIZES 64 128 256 512 CACHE STRING "Block sizes to build as separate test executables") set(PERNIX_TEST_TARGETS) @@ -17,6 +18,15 @@ foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) target_compile_options(${TEST_TARGET} PRIVATE ${GTEST_CFLAGS}) target_compile_definitions(${TEST_TARGET} PRIVATE PERNIX_TEST_BLOCK_SIZE=${BLOCK_SIZE}) target_include_directories(${TEST_TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) + + set(HEADER_ONLY_TARGET pernix_header_only_tests_${BLOCK_SIZE}) + list(APPEND PERNIX_TEST_TARGETS ${HEADER_ONLY_TARGET}) + add_executable(${HEADER_ONLY_TARGET} header_only_tests.cpp main.cpp) + target_link_libraries(${HEADER_ONLY_TARGET} PRIVATE pernix_header_only ${GTEST_LDFLAGS}) + target_compile_features(${HEADER_ONLY_TARGET} PRIVATE cxx_std_20) + target_compile_options(${HEADER_ONLY_TARGET} PRIVATE ${GTEST_CFLAGS}) + target_compile_definitions(${HEADER_ONLY_TARGET} PRIVATE PERNIX_TEST_BLOCK_SIZE=${BLOCK_SIZE}) + target_include_directories(${HEADER_ONLY_TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) endforeach () add_custom_target(pernix_tests DEPENDS ${PERNIX_TEST_TARGETS}) @@ -29,6 +39,11 @@ foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) NO_PRETTY_TYPES PROPERTIES LABELS "unit;block_size_${BLOCK_SIZE}" ) + gtest_discover_tests(pernix_header_only_tests_${BLOCK_SIZE} + DISCOVERY_MODE PRE_TEST + EXTRA_ARGS --gtest_color=yes + PROPERTIES LABELS "unit;header_only;block_size_${BLOCK_SIZE}" + ) endforeach () if (PERNIX_ENABLE_CODE_COVERAGE) diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp index abb502b..ef57986 100644 --- a/tests/fallback_tests.cpp +++ b/tests/fallback_tests.cpp @@ -7,6 +7,16 @@ #include #include +#if defined(__linux__) && defined(__aarch64__) +#include +#ifndef HWCAP_SVE +#define HWCAP_SVE (1 << 22) +#endif +#ifndef HWCAP2_SVE2 +#define HWCAP2_SVE2 (1 << 1) +#endif +#endif + // --------------------------------------------------------------------------- // Fallback compress: verify byte-exact match against the reference // --------------------------------------------------------------------------- @@ -314,3 +324,74 @@ TEST(ErrorCodeTest, NullPointerReturnsError) { st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, 64, src, 1.0f, nullptr); EXPECT_EQ(st, PERNIX_STATUS_INVALID_ARGUMENT); } + +TEST(ErrorCodeTest, FallbackAliasMatchesScalar) { + EXPECT_EQ(PERNIX_BACKEND_FALLBACK, PERNIX_BACKEND_FALLBACK_SCALAR); +} + +TEST(ErrorCodeTest, FallbackStdparReturnsUnsupportedImplementationByDefault) { + constexpr u32 BS = 64; + constexpr u8 BW = 8; + + f32 src[(BS * 8U) / BW] = {}; + u8 dst[BS] = {}; + + const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +} + +TEST(ErrorCodeTest, FallbackSimdReturnsUnsupportedImplementationByDefault) { + constexpr u32 BS = 64; + constexpr u8 BW = 8; + + f32 src[(BS * 8U) / BW] = {}; + u8 dst[BS] = {}; + + const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_SIMD, BW, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +} + +TEST(ErrorCodeTest, UnsupportedImplementationReturnsError) { + constexpr u32 BS = 64; + constexpr u8 BW = 8; + + f32 src[(BS * 8U) / BW] = {}; + u8 dst[BS] = {}; + + pernix_backend backend = PERNIX_BACKEND_FALLBACK; + bool found = false; + +#if defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) + if (!__builtin_cpu_supports("avx512vbmi")) { + backend = PERNIX_BACKEND_X86_AVX512_VBMI; + found = true; + } else if (!(__builtin_cpu_supports("avx2") && __builtin_cpu_supports("bmi2"))) { + backend = PERNIX_BACKEND_X86_BMI2; + found = true; + } else if (!__builtin_cpu_supports("avx2")) { + backend = PERNIX_BACKEND_X86_AVX2; + found = true; + } +#elif defined(__aarch64__) || defined(_M_ARM64) + const bool neon = true; +#if defined(__linux__) && defined(__aarch64__) + const bool sve2 = (getauxval(AT_HWCAP2) & HWCAP2_SVE2) != 0; +#else + const bool sve2 = false; +#endif + if (!sve2) { + backend = PERNIX_BACKEND_ARM64_SVE; + found = true; + } else if (!neon) { + backend = PERNIX_BACKEND_ARM64_NEON; + found = true; + } +#endif + + if (!found) { + GTEST_SKIP() << "No compiled explicit backend is unsupported on this machine"; + } + + const auto st = pernix_compress_block_f32(backend, BW, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +} diff --git a/tests/header_only_tests.cpp b/tests/header_only_tests.cpp new file mode 100644 index 0000000..211667c --- /dev/null +++ b/tests/header_only_tests.cpp @@ -0,0 +1,69 @@ +#include + +#include + +#include +#include + +namespace { +constexpr u8 kBitWidth = 8; +constexpr u32 kBlockSize = PERNIX_TEST_BLOCK_SIZE; +constexpr u32 kBlockElements = (kBlockSize * 8U) / kBitWidth; + +template +std::vector make_input() { + std::vector values(kBlockElements); + for (u32 i = 0; i < kBlockElements; ++i) { + values[static_cast(i)] = static_cast((static_cast(i % 13U) - 6) * 0.25); + } + return values; +} + +template +void expect_round_trip(const pernix::Backend backend) { + auto input = make_input(); + std::vector compressed(kBlockSize); + std::vector restored(kBlockElements, static_cast(0)); + + ASSERT_EQ(pernix::compress_block(backend, kBitWidth, kBlockSize, std::span(input), + static_cast(1), std::span(compressed)), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_block(backend, kBitWidth, kBlockSize, std::span(compressed), + static_cast(1), std::span(restored)), + PERNIX_STATUS_OK); + + for (usize i = 0; i < restored.size(); ++i) { + EXPECT_NEAR(restored[i], input[i], static_cast(1.0)); + } +} +} + +TEST(HeaderOnlyPernix, FallbackFloatRoundTrip) { + expect_round_trip(pernix::Backend::Fallback); +} + +TEST(HeaderOnlyPernix, FallbackAliasMatchesScalar) { + EXPECT_EQ(static_cast(pernix::Backend::Fallback), static_cast(pernix::Backend::FallbackScalar)); +} + +TEST(HeaderOnlyPernix, FallbackStdparReturnsUnsupportedImplementation) { + auto input = make_input(); + std::vector compressed(kBlockSize); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, kBitWidth, kBlockSize, + std::span(input), 1.0f, std::span(compressed)), + PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +} + +TEST(HeaderOnlyPernix, FallbackSimdReturnsUnsupportedImplementation) { + auto input = make_input(); + std::vector compressed(kBlockSize); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackSimd, kBitWidth, kBlockSize, + std::span(input), 1.0f, std::span(compressed)), + PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +} + +TEST(HeaderOnlyPernix, AutoDoubleRoundTrip) { + expect_round_trip(pernix::Backend::Auto); +} From 88ce357deb7e9346e2ce52bcced8bbaa499e2185 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Sat, 27 Jun 2026 15:49:06 +0200 Subject: [PATCH 16/43] refactor: clean up code formatting and improve consistency across header files --- include/pernix/arm64/neon/common.h | 362 ++--- include/pernix/arm64/neon/compression.h | 204 +-- include/pernix/arm64/neon/tables.h | 360 ++--- include/pernix/arm64/neon/unpacking.h | 128 +- include/pernix/arm64/sve2/compression.h | 73 +- include/pernix/fallback/scalar_compression.h | 268 ++-- .../pernix/fallback/scalar_decompression.h | 235 ++- include/pernix/x86/avx2/avx2_compression.h | 791 +++++----- include/pernix/x86/avx2/avx2_decompression.h | 605 ++++---- .../x86/avx512vbmi/avx512vbmi_compression.h | 1327 +++++++++-------- .../x86/avx512vbmi/avx512vbmi_decompression.h | 1083 +++++++------- include/pernix/x86/avx512vbmi/compat.h | 457 +++--- include/pernix/x86/avx512vbmi/packing.h | 621 ++++---- include/pernix/x86/avx512vbmi/unpacking.h | 835 +++++------ include/pernix/x86/bmi2/bmi2_compression.h | 437 +++--- include/pernix/x86/bmi2/bmi2_decompression.h | 657 ++++---- include/pernix/x86/utils.h | 12 +- 17 files changed, 4230 insertions(+), 4225 deletions(-) diff --git a/include/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h index 6cd9ad0..cda7065 100644 --- a/include/pernix/arm64/neon/common.h +++ b/include/pernix/arm64/neon/common.h @@ -6,218 +6,218 @@ #include namespace pernix::arm64::neon::internal { - struct float64x2x8_t { - float64x2_t val[8]; +struct float64x2x8_t { + float64x2_t val[8]; +}; + +static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements) { + const u32 tail_bits = remaining_elements * bit_width; + const u32 tail_bytes = (tail_bits + 7u) / 8u; + return tail_bytes; +} + +__always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(const int8x16_t& input) { + const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); + const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); + + return { + { + vmovl_s16(vget_low_s16(s16_lo)), + vmovl_s16(vget_high_s16(s16_lo)), + vmovl_s16(vget_low_s16(s16_hi)), + vmovl_s16(vget_high_s16(s16_hi)), + } }; +} - static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements) { - const u32 tail_bits = remaining_elements * bit_width; - const u32 tail_bytes = (tail_bits + 7u) / 8u; - return tail_bytes; - } - -__always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(const int8x16_t &input) { - const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); - const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); - - return { - { - vmovl_s16(vget_low_s16(s16_lo)), - vmovl_s16(vget_high_s16(s16_lo)), - vmovl_s16(vget_low_s16(s16_hi)), - vmovl_s16(vget_high_s16(s16_hi)), - } - }; - } - -__always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(const int16x8_t &input) { - return { - { - vmovl_s16(vget_low_s16(input)), - vmovl_s16(vget_high_s16(input)), - } - }; - } - -__always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t &input, const float32x4_t &scale) { - return { - { - vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), - } - }; - } - -__always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t &input, const float32x4_t &scale) { - return { - { - vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), - } - }; - } - -__always_inline float32x4_t neon_dequantize_epi32(const int32x4_t &input, const float32x4_t &scale) { - return vmulq_f32(vcvtq_f32_s32(input), scale); - } - -__always_inline float64x2_t neon_dequantize_epi32_f64(const int32x2_t &input, const float64x2_t &scale) { - return vmulq_f64(vcvtq_f64_s64(vmovl_s32(input)), scale); - } - -__always_inline float64x2x2_t neon_dequantize_epi32_f64(const int32x4_t &input, const float64x2_t &scale) { - return { - { - neon_dequantize_epi32_f64(vget_low_s32(input), scale), - neon_dequantize_epi32_f64(vget_high_s32(input), scale), - } - }; - } - -__always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t &input, const float64x2_t &scale) { - const float64x2x2_t dequantized_low = neon_dequantize_epi32_f64(input.val[0], scale); - const float64x2x2_t dequantized_high = neon_dequantize_epi32_f64(input.val[1], scale); - - return { - { - dequantized_low.val[0], - dequantized_low.val[1], - dequantized_high.val[0], - dequantized_high.val[1], - } - }; - } - -__always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t &input, const float64x2_t &scale) { - const float64x2x2_t dequantized0 = neon_dequantize_epi32_f64(input.val[0], scale); - const float64x2x2_t dequantized1 = neon_dequantize_epi32_f64(input.val[1], scale); - const float64x2x2_t dequantized2 = neon_dequantize_epi32_f64(input.val[2], scale); - const float64x2x2_t dequantized3 = neon_dequantize_epi32_f64(input.val[3], scale); - - return { - { - dequantized0.val[0], - dequantized0.val[1], - dequantized1.val[0], - dequantized1.val[1], - dequantized2.val[0], - dequantized2.val[1], - dequantized3.val[0], - dequantized3.val[1], - } - }; - } - -__always_inline uint8x16_t neon_load_tail_elements_int8(const u8 *input, const u32 tail_bytes_count) { - u8 buffer[16] = {0}; - std::memcpy(buffer, input, tail_bytes_count); - return vld1q_u8(buffer); - } - -__always_inline uint16x8_t neon_load_tail_elements_int16(const u8 *input, const u32 tail_bytes_count) { - u16 buffer[8] = {0}; - std::memcpy(buffer, input, tail_bytes_count); - return vld1q_u16(buffer); - } +__always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(const int16x8_t& input) { + return { + { + vmovl_s16(vget_low_s16(input)), + vmovl_s16(vget_high_s16(input)), + } + }; +} + +__always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, const float32x4_t& scale) { + return { + { + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), + } + }; +} -__always_inline uint32x4_t neon_load_tail_elements_int32(const u8 *input, const u32 tail_bytes_count) { - u32 buffer[4] = {0}; - std::memcpy(buffer, input, tail_bytes_count); - return vld1q_u32(buffer); - } +__always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t& input, const float32x4_t& scale) { + return { + { + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + } + }; +} -__always_inline float32x4_t neon_load_tail_elements_f32(const u8 *input, const u32 tail_elements) { - float32_t buffer[4] = {0.0f}; - std::memcpy(buffer, input, tail_elements * sizeof(float32_t)); - return vld1q_f32(buffer); - } +__always_inline float32x4_t neon_dequantize_epi32(const int32x4_t& input, const float32x4_t& scale) { + return vmulq_f32(vcvtq_f32_s32(input), scale); +} -__always_inline float64x2_t neon_load_tail_elements_f64(const u8 *input, const u32 tail_elements) { - float64_t buffer[2] = {0.0}; - std::memcpy(buffer, input, tail_elements * sizeof(float64_t)); - return vld1q_f64(buffer); - } +__always_inline float64x2_t neon_dequantize_epi32_f64(const int32x2_t& input, const float64x2_t& scale) { + return vmulq_f64(vcvtq_f64_s64(vmovl_s32(input)), scale); +} -__always_inline void neon_store_tail_elements_int8(u8 *output, const uint8x16x4_t &data, - const u32 tail_elements) { - u8 buffer[16 * 4]; - for (u32 i = 0; i < 4; ++i) { - vst1q_u8(buffer + i * 16, data.val[i]); +__always_inline float64x2x2_t neon_dequantize_epi32_f64(const int32x4_t& input, const float64x2_t& scale) { + return { + { + neon_dequantize_epi32_f64(vget_low_s32(input), scale), + neon_dequantize_epi32_f64(vget_high_s32(input), scale), + } + }; +} + +__always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t& input, const float64x2_t& scale) { + const float64x2x2_t dequantized_low = neon_dequantize_epi32_f64(input.val[0], scale); + const float64x2x2_t dequantized_high = neon_dequantize_epi32_f64(input.val[1], scale); + + return { + { + dequantized_low.val[0], + dequantized_low.val[1], + dequantized_high.val[0], + dequantized_high.val[1], } - std::memcpy(output, buffer, tail_elements * sizeof(u8)); + }; +} + +__always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input, const float64x2_t& scale) { + const float64x2x2_t dequantized0 = neon_dequantize_epi32_f64(input.val[0], scale); + const float64x2x2_t dequantized1 = neon_dequantize_epi32_f64(input.val[1], scale); + const float64x2x2_t dequantized2 = neon_dequantize_epi32_f64(input.val[2], scale); + const float64x2x2_t dequantized3 = neon_dequantize_epi32_f64(input.val[3], scale); + + return { + { + dequantized0.val[0], + dequantized0.val[1], + dequantized1.val[0], + dequantized1.val[1], + dequantized2.val[0], + dequantized2.val[1], + dequantized3.val[0], + dequantized3.val[1], + } + }; +} + +__always_inline uint8x16_t neon_load_tail_elements_int8(const u8* input, const u32 tail_bytes_count) { + u8 buffer[16] = {0}; + std::memcpy(buffer, input, tail_bytes_count); + return vld1q_u8(buffer); +} + +__always_inline uint16x8_t neon_load_tail_elements_int16(const u8* input, const u32 tail_bytes_count) { + u16 buffer[8] = {0}; + std::memcpy(buffer, input, tail_bytes_count); + return vld1q_u16(buffer); +} + +__always_inline uint32x4_t neon_load_tail_elements_int32(const u8* input, const u32 tail_bytes_count) { + u32 buffer[4] = {0}; + std::memcpy(buffer, input, tail_bytes_count); + return vld1q_u32(buffer); +} + +__always_inline float32x4_t neon_load_tail_elements_f32(const u8* input, const u32 tail_elements) { + float32_t buffer[4] = {0.0f}; + std::memcpy(buffer, input, tail_elements * sizeof(float32_t)); + return vld1q_f32(buffer); +} + +__always_inline float64x2_t neon_load_tail_elements_f64(const u8* input, const u32 tail_elements) { + float64_t buffer[2] = {0.0}; + std::memcpy(buffer, input, tail_elements * sizeof(float64_t)); + return vld1q_f64(buffer); +} + +__always_inline void neon_store_tail_elements_int8(u8* output, const uint8x16x4_t& data, + const u32 tail_elements) { + u8 buffer[16 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_u8(buffer + i * 16, data.val[i]); } + std::memcpy(output, buffer, tail_elements * sizeof(u8)); +} -__always_inline void neon_store_tail_elements_int16(u16 *output, const uint16x8x4_t &data, +__always_inline void neon_store_tail_elements_int16(u16* output, const uint16x8x4_t& data, const u32 tail_elements) { - u16 buffer[8 * 4]; - for (u32 i = 0; i < 4; ++i) { - vst1q_u16(buffer + i * 8, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(u16)); + u16 buffer[8 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_u16(buffer + i * 8, data.val[i]); } + std::memcpy(output, buffer, tail_elements * sizeof(u16)); +} -__always_inline void neon_store_tail_elements_int32(u32 *output, const uint32x4x4_t &data, +__always_inline void neon_store_tail_elements_int32(u32* output, const uint32x4x4_t& data, const u32 tail_elements) { - u32 buffer[4 * 4]; - for (u32 i = 0; i < 4; ++i) { - vst1q_u32(buffer + i * 4, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(u32)); + u32 buffer[4 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_u32(buffer + i * 4, data.val[i]); } + std::memcpy(output, buffer, tail_elements * sizeof(u32)); +} -__always_inline void neon_store_tail_elements_f32(float32_t *output, const float32x4x4_t &data, +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x4_t& data, const u32 tail_elements) { - float32_t buffer[16 * 4]; - for (u32 i = 0; i < 4; ++i) { - vst1q_f32(buffer + i * 4, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); + float32_t buffer[16 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_f32(buffer + i * 4, data.val[i]); } + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); +} -__always_inline void neon_store_tail_elements_f32(float32_t *output, const float32x4x2_t &data, +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x2_t& data, const u32 tail_elements) { - float32_t buffer[8 * 2]; - for (u32 i = 0; i < 2; ++i) { - vst1q_f32(buffer + i * 4, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); + float32_t buffer[8 * 2]; + for (u32 i = 0; i < 2; ++i) { + vst1q_f32(buffer + i * 4, data.val[i]); } + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); +} -__always_inline void neon_store_tail_elements_f32(float32_t *output, const float32x4_t &data, +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4_t& data, const u32 tail_elements) { - float32_t buffer[4]; - vst1q_f32(buffer, data); - std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); - } + float32_t buffer[4]; + vst1q_f32(buffer, data); + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); +} -__always_inline void neon_store_tail_elements_f64(float64_t *output, const float64x2x4_t &data, +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x4_t& data, const u32 tail_elements) { - float64_t buffer[2 * 4]; - for (u32 i = 0; i < 4; ++i) { - vst1q_f64(buffer + i * 2, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); + float64_t buffer[2 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); +} -__always_inline void neon_store_tail_elements_f64(float64_t *output, const float64x2x2_t &data, +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x2_t& data, const u32 tail_elements) { - float64_t buffer[2 * 2]; - for (u32 i = 0; i < 2; ++i) { - vst1q_f64(buffer + i * 2, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); + float64_t buffer[2 * 2]; + for (u32 i = 0; i < 2; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); +} -__always_inline void neon_store_tail_elements_f64(float64_t *output, const float64x2x8_t &data, +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x8_t& data, const u32 tail_elements) { - float64_t buffer[2 * 8]; - for (u32 i = 0; i < 8; ++i) { - vst1q_f64(buffer + i * 2, data.val[i]); - } - std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); + float64_t buffer[2 * 8]; + for (u32 i = 0; i < 8; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); +} } // namespace pernix::arm64::neon::internal #endif // PERNIX_ARM64_NEON_COMMON_H diff --git a/include/pernix/arm64/neon/compression.h b/include/pernix/arm64/neon/compression.h index 83d46c3..3a03199 100644 --- a/include/pernix/arm64/neon/compression.h +++ b/include/pernix/arm64/neon/compression.h @@ -8,114 +8,114 @@ #include namespace pernix::arm64::neon { - namespace internal { - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_1to8(const u8 * __restrict__ input, const f32 scale, - f32 * __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; - } - - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_9to16(const u8 * __restrict__ input, const f32 scale, - f32 * __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; - } - - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_17to24(const u8 * __restrict__ input, const f32 scale, - f32 * __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; - } - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_1to8(const u8 * __restrict__ input, const f64 scale, - f64 * __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; - } - - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_9to16(const u8 * __restrict__ input, const f64 scale, - f64 * __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; - } - - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_17to24(const u8 * __restrict__ input, const f64 scale, - f64 * __restrict__ output) { - static_assert(true, "Not yet implemented"); - return -1; - } - } // namespace internal - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block(const u8 * __restrict__ input, const f32 scale, - f32 * __restrict__ output) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::neon_compress_block_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::neon_compress_block_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::neon_compress_block_17to24(input, scale, output); - } - return 0; +namespace internal { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_1to8(const u8* __restrict__ input, const f32 scale, + f32* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_9to16(const u8* __restrict__ input, const f32 scale, + f32* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_17to24(const u8* __restrict__ input, const f32 scale, + f32* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_1to8(const u8* __restrict__ input, const f64 scale, + f64* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_9to16(const u8* __restrict__ input, const f64 scale, + f64* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_17to24(const u8* __restrict__ input, const f64 scale, + f64* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block(const u8* __restrict__ input, const f32 scale, + f32* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_compress_block_17to24(input, scale, output); } - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block(const u8 * __restrict__ input, const f64 scale, - f64 * __restrict__ output) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::neon_compress_block_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::neon_compress_block_9to16(input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::neon_compress_block_17to24(input, scale, output); - } - return 0; + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block(const u8* __restrict__ input, const f64 scale, + f64* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_compress_block_17to24(input, scale, output); + } + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_compress_blocks(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output, + const u32 blocks) { + const u8* block_input = input; + f32* block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + neon_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int neon_compress_blocks(const u8 * __restrict__ input, const f32 scale, f32 * __restrict__ output, - const u32 blocks) { - const u8 *block_input = input; - f32 *block_output = output; - - for (u32 block = 0; block < blocks; ++block) { - neon_compress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + return 0; +} - return 0; - } +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_compress_blocks(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output, + const u32 blocks) { + const u8* block_input = input; + f64* block_output = output; - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int neon_compress_blocks(const u8 * __restrict__ input, const f64 scale, f64 * __restrict__ output, - const u32 blocks) { - const u8 *block_input = input; - f64 *block_output = output; - - for (u32 block = 0; block < blocks; ++block) { - neon_compress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - return 0; + for (u32 block = 0; block < blocks; ++block) { + neon_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } + return 0; +} } // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_COMPRESSION_H diff --git a/include/pernix/arm64/neon/tables.h b/include/pernix/arm64/neon/tables.h index beb7c47..2cf4e37 100644 --- a/include/pernix/arm64/neon/tables.h +++ b/include/pernix/arm64/neon/tables.h @@ -7,208 +7,208 @@ #include namespace pernix::arm64::neon::internal { - namespace detail { - inline constexpr std::size_t neon_vector_width = 128; - inline constexpr u8 inactive_lane = 0xff; - - template - constexpr bool table_indices_are_valid(const std::array &table) { - return std::ranges::all_of(table, [](const u8 index) { - return index == inactive_lane || index < Elements; - }); +namespace detail { +inline constexpr std::size_t neon_vector_width = 128; +inline constexpr u8 inactive_lane = 0xff; + +template +constexpr bool table_indices_are_valid(const std::array& table) { + return std::ranges::all_of(table, [](const u8 index) { + return index == inactive_lane || index < Elements; + }); +} + +template +constexpr std::array make_primary_permute() { + static_assert(LANE_BITS % 8 == 0); + + constexpr std::size_t lane_bytes = LANE_BITS / 8; + static_assert(ELEMENTS % lane_bytes == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t first_byte = bit_start / 8; + const std::size_t base = entry * lane_bytes; + + for (std::size_t lane_byte = 0; lane_byte < lane_bytes; ++lane_byte) { + table[base + lane_byte] = static_cast(first_byte + lane_byte); } + } - template - constexpr std::array make_primary_permute() { - static_assert(LANE_BITS % 8 == 0); + return table; +} - constexpr std::size_t lane_bytes = LANE_BITS / 8; - static_assert(ELEMENTS % lane_bytes == 0); +template +constexpr std::array make_spill_permute() { + static_assert(LANE_BITS % 8 == 0); - std::array table{}; - table.fill(inactive_lane); + constexpr std::size_t lane_bytes = LANE_BITS / 8; + static_assert(ELEMENTS % lane_bytes == 0); - for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t first_byte = bit_start / 8; - const std::size_t base = entry * lane_bytes; + std::array table{}; + table.fill(inactive_lane); - for (std::size_t lane_byte = 0; lane_byte < lane_bytes; ++lane_byte) { - table[base + lane_byte] = static_cast(first_byte + lane_byte); - } - } + for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t first_byte = bit_start / 8; + const std::size_t bit_offset = bit_start % 8; + const std::size_t base = entry * lane_bytes; - return table; + if (bit_offset + BIT_WIDTH > LANE_BITS) { + table[base] = static_cast(first_byte + lane_bytes); } + } - template - constexpr std::array make_spill_permute() { - static_assert(LANE_BITS % 8 == 0); + return table; +} - constexpr std::size_t lane_bytes = LANE_BITS / 8; - static_assert(ELEMENTS % lane_bytes == 0); +template +constexpr std::array make_shift_right() { + std::array table{}; + table.fill(0); - std::array table{}; - table.fill(inactive_lane); + for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t bit_offset = bit_start % 8u; - for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t first_byte = bit_start / 8; - const std::size_t bit_offset = bit_start % 8; - const std::size_t base = entry * lane_bytes; + table[entry] = -static_cast(bit_offset); + } - if (bit_offset + BIT_WIDTH > LANE_BITS) { - table[base] = static_cast(first_byte + lane_bytes); - } - } + return table; +} - return table; - } - - template - constexpr std::array make_shift_right() { - std::array table{}; - table.fill(0); - - for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t bit_offset = bit_start % 8u; - - table[entry] = -static_cast(bit_offset); - } - - return table; - } - - template - constexpr std::array make_shift_left_for_spill() { - std::array table{}; - table.fill(0); +template +constexpr std::array make_shift_left_for_spill() { + std::array table{}; + table.fill(0); - for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t bit_offset = bit_start % 8u; - const bool spills = bit_offset + BIT_WIDTH > LANE_BITS; + for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { + const std::size_t bit_start = entry * BIT_WIDTH; + const std::size_t bit_offset = bit_start % 8u; + const bool spills = bit_offset + BIT_WIDTH > LANE_BITS; - table[entry] = spills ? static_cast(LANE_BITS - bit_offset) : 0; - } + table[entry] = spills ? static_cast(LANE_BITS - bit_offset) : 0; + } - return table; - } - - template - constexpr std::array make_contiguous_permute_32() { - static_assert(ELEMENTS % 4 == 0); - - std::array table{}; - table.fill(inactive_lane); - - for (std::size_t entry = 0; entry < ELEMENTS / 4; ++entry) { - const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; - const std::size_t bit_end = bit_start + BIT_WIDTH - 1; - const std::size_t first_byte = bit_start / 8; - const std::size_t last_byte = bit_end / 8; - const std::size_t base = entry * 4; - - for (std::size_t byte = first_byte; byte <= last_byte; ++byte) { - table[base + (byte - first_byte)] = static_cast(byte); - } - } - - return table; - } + return table; +} - template - constexpr std::array make_shift_right_32() { - std::array table{}; - table.fill(0); +template +constexpr std::array make_contiguous_permute_32() { + static_assert(ELEMENTS % 4 == 0); - for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { - const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + std::array table{}; + table.fill(inactive_lane); - table[entry] = -static_cast(bit_start % 8u); - } + for (std::size_t entry = 0; entry < ELEMENTS / 4; ++entry) { + const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + const std::size_t bit_end = bit_start + BIT_WIDTH - 1; + const std::size_t first_byte = bit_start / 8; + const std::size_t last_byte = bit_end / 8; + const std::size_t base = entry * 4; - return table; + for (std::size_t byte = first_byte; byte <= last_byte; ++byte) { + table[base + (byte - first_byte)] = static_cast(byte); } - } // namespace detail - - template - struct table_unpacking; - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && VECTOR_WIDTH == detail::neon_vector_width) - struct table_unpacking { - private: - static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; - static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 8; - - public: - static constexpr u8 bit_width = BIT_WIDTH; - - alignas(64) static constexpr std::array permute1 = - detail::make_primary_permute(); - alignas(64) static constexpr std::array permute2 = - detail::make_spill_permute(); - alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); - alignas(64) static constexpr std::array shift2 = - detail::make_shift_left_for_spill(); - - static_assert(PERMUTE_ELEMENTS == 16); - static_assert(SHIFT_ELEMENTS == 16); - static_assert(detail::table_indices_are_valid(permute1)); - static_assert(detail::table_indices_are_valid(permute2)); - }; - - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && VECTOR_WIDTH == detail::neon_vector_width) - struct table_unpacking { - private: - static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; - static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 16; - - public: - static constexpr u8 bit_width = BIT_WIDTH; - - alignas(64) static constexpr std::array permute1 = - detail::make_primary_permute(); - alignas(64) static constexpr std::array permute2 = - detail::make_spill_permute(); - alignas(64) static constexpr std::array shift1 = - detail::make_shift_right(); - alignas(64) static constexpr std::array shift2 = - detail::make_shift_left_for_spill(); - - static_assert(PERMUTE_ELEMENTS == 16); - static_assert(SHIFT_ELEMENTS == 8); - static_assert(detail::table_indices_are_valid(permute1)); - static_assert(detail::table_indices_are_valid(permute2)); - }; - - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && VECTOR_WIDTH == detail::neon_vector_width && START_BIT_OFFSET < - 8) - struct table_unpacking { - private: - static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; - static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 32; - - public: - static constexpr u8 bit_width = BIT_WIDTH; - - alignas(64) static constexpr std::array permute = - detail::make_contiguous_permute_32(); - alignas(64) static constexpr std::array shift = - detail::make_shift_right_32(); - - static_assert(PERMUTE_ELEMENTS == 16); - static_assert(SHIFT_ELEMENTS == 4); - static_assert(detail::table_indices_are_valid(permute)); - }; - - template - struct table_packing; + } + + return table; +} + +template +constexpr std::array make_shift_right_32() { + std::array table{}; + table.fill(0); + + for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { + const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + + table[entry] = -static_cast(bit_start % 8u); + } + + return table; +} +} // namespace detail + +template +struct table_unpacking; + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && VECTOR_WIDTH == detail::neon_vector_width) +struct table_unpacking { +private: + static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 8; + +public: + static constexpr u8 bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute1 = + detail::make_primary_permute(); + alignas(64) static constexpr std::array permute2 = + detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); + alignas(64) static constexpr std::array shift2 = + detail::make_shift_left_for_spill(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 16); + static_assert(detail::table_indices_are_valid(permute1)); + static_assert(detail::table_indices_are_valid(permute2)); +}; + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && VECTOR_WIDTH == detail::neon_vector_width) +struct table_unpacking { +private: + static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 16; + +public: + static constexpr u8 bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute1 = + detail::make_primary_permute(); + alignas(64) static constexpr std::array permute2 = + detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = + detail::make_shift_right(); + alignas(64) static constexpr std::array shift2 = + detail::make_shift_left_for_spill(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 8); + static_assert(detail::table_indices_are_valid(permute1)); + static_assert(detail::table_indices_are_valid(permute2)); +}; + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && VECTOR_WIDTH == detail::neon_vector_width && START_BIT_OFFSET < + 8) +struct table_unpacking { +private: + static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 32; + +public: + static constexpr u8 bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute = + detail::make_contiguous_permute_32(); + alignas(64) static constexpr std::array shift = + detail::make_shift_right_32(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 4); + static_assert(detail::table_indices_are_valid(permute)); +}; + +template +struct table_packing; } // namespace pernix::arm64::internal #endif // PERNIX_ARM64_NEON_TABLES_H diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h index cc9bcf0..1d227c1 100644 --- a/include/pernix/arm64/neon/unpacking.h +++ b/include/pernix/arm64/neon/unpacking.h @@ -7,95 +7,95 @@ using namespace pernix::arm64::neon::internal; namespace pernix::arm64::neon::internal::b128 { - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t &input) { - if constexpr (BIT_WIDTH == 8) { - return vreinterpretq_s8_u8(input); - } else if constexpr (BIT_WIDTH == 1) { - using tables = table_unpacking; - - const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); - const uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); - - return vreinterpretq_s8_u8(vandq_u8(shifted, vdupq_n_u8(1))); - } else { - using tables = table_unpacking; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t& input) { + if constexpr (BIT_WIDTH == 8) { + return vreinterpretq_s8_u8(input); + } else if constexpr (BIT_WIDTH == 1) { + using tables = table_unpacking; - const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); + const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); + const uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); - uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); + return vreinterpretq_s8_u8(vandq_u8(shifted, vdupq_n_u8(1))); + } else { + using tables = table_unpacking; - if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { - const uint8x16_t permuted2_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute2.data())); + const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); - shifted = vorrq_u8(shifted, vshlq_u8(permuted2_u8, vld1q_s8(tables::shift2.data()))); - } + uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); - constexpr int shift = 8 - BIT_WIDTH; - shifted = vshlq_n_u8(shifted, shift); + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + const uint8x16_t permuted2_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute2.data())); - if constexpr (SIGN_VALUES) { - return vshlq_s8(vreinterpretq_s8_u8(shifted), vdupq_n_s8(-shift)); - } else { - return vreinterpretq_s8_u8(vshlq_u8(shifted, vdupq_n_s8(-shift))); - } + shifted = vorrq_u8(shifted, vshlq_u8(permuted2_u8, vld1q_s8(tables::shift2.data()))); } - } - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t &input) { - if constexpr (BIT_WIDTH == 16) { - return vreinterpretq_s16_u16(input); + constexpr int shift = 8 - BIT_WIDTH; + shifted = vshlq_n_u8(shifted, shift); + + if constexpr (SIGN_VALUES) { + return vshlq_s8(vreinterpretq_s8_u8(shifted), vdupq_n_s8(-shift)); } else { - using tables = table_unpacking; + return vreinterpretq_s8_u8(vshlq_u8(shifted, vdupq_n_s8(-shift))); + } + } +} - const uint8x16_t input_u8 = vreinterpretq_u8_u16(input); +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t& input) { + if constexpr (BIT_WIDTH == 16) { + return vreinterpretq_s16_u16(input); + } else { + using tables = table_unpacking; - const uint8x16_t permuted1_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute1.data())); + const uint8x16_t input_u8 = vreinterpretq_u8_u16(input); - uint16x8_t shifted = vshlq_u16(vreinterpretq_u16_u8(permuted1_u8), vld1q_s16(tables::shift1.data())); + const uint8x16_t permuted1_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute1.data())); - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const uint8x16_t permuted2_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute2.data())); + uint16x8_t shifted = vshlq_u16(vreinterpretq_u16_u8(permuted1_u8), vld1q_s16(tables::shift1.data())); - const uint16x8_t shifted2 = vshlq_u16(vreinterpretq_u16_u8(permuted2_u8), - vld1q_s16(tables::shift2.data())); + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const uint8x16_t permuted2_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute2.data())); - shifted = vorrq_u16(shifted, shifted2); - } + const uint16x8_t shifted2 = vshlq_u16(vreinterpretq_u16_u8(permuted2_u8), + vld1q_s16(tables::shift2.data())); - constexpr int shift = 16 - BIT_WIDTH; - shifted = vshlq_n_u16(shifted, shift); + shifted = vorrq_u16(shifted, shifted2); + } - if constexpr (SIGN_VALUES) { - return vshlq_s16(vreinterpretq_s16_u16(shifted), vdupq_n_s16(-shift)); - } else { - return vreinterpretq_s16_u16(vshlq_u16(shifted, vdupq_n_s16(-shift))); - } + constexpr int shift = 16 - BIT_WIDTH; + shifted = vshlq_n_u16(shifted, shift); + + if constexpr (SIGN_VALUES) { + return vshlq_s16(vreinterpretq_s16_u16(shifted), vdupq_n_s16(-shift)); + } else { + return vreinterpretq_s16_u16(vshlq_u16(shifted, vdupq_n_s16(-shift))); } } +} - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline int32x4_t neon_unpack_epi32_17to24(const uint32x4_t &input) { - using tables = table_unpacking; +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline int32x4_t neon_unpack_epi32_17to24(const uint32x4_t& input) { + using tables = table_unpacking; - const uint8x16_t input_8 = vreinterpretq_u8_u32(input); + const uint8x16_t input_8 = vreinterpretq_u8_u32(input); - const uint8x16_t permuted_u8 = vqtbl1q_u8(input_8, vld1q_u8(tables::permute.data())); + const uint8x16_t permuted_u8 = vqtbl1q_u8(input_8, vld1q_u8(tables::permute.data())); - const uint32x4_t value = vshlq_u32(vreinterpretq_u32_u8(permuted_u8), vld1q_s32(tables::shift.data())); + const uint32x4_t value = vshlq_u32(vreinterpretq_u32_u8(permuted_u8), vld1q_s32(tables::shift.data())); - if constexpr (SIGN_VALUES) { - constexpr int sign_shift = 32 - BIT_WIDTH; - return vshrq_n_s32(vreinterpretq_s32_u32(vshlq_n_u32(value, sign_shift)), sign_shift); - } else { - constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1u; - return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); - } + if constexpr (SIGN_VALUES) { + constexpr int sign_shift = 32 - BIT_WIDTH; + return vshrq_n_s32(vreinterpretq_s32_u32(vshlq_n_u32(value, sign_shift)), sign_shift); + } else { + constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1u; + return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); } +} } // namespace pernix::arm64::neon::internal::b128 #endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/arm64/sve2/compression.h b/include/pernix/arm64/sve2/compression.h index 33f48fe..e441f22 100644 --- a/include/pernix/arm64/sve2/compression.h +++ b/include/pernix/arm64/sve2/compression.h @@ -7,42 +7,43 @@ #include namespace pernix { - namespace internal { - template - inline constexpr bool sve2_compression_unimplemented_v = false; - } // namespace internal - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int sve2_compress_block(const f32 *, f32, u8 *) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); - return -1; - } - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int sve2_compress_block(const f64 *, f64, u8 *) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); - return -1; - } - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int sve2_compress_blocks(const f32 *, f32, u8 *, u32) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); - return -1; - } - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int sve2_compress_blocks(const f64 *, f64, u8 *, u32) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); - return -1; - } +namespace internal { +template +inline constexpr bool sve2_compression_unimplemented_v = false; +} // namespace internal + + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_block(const f32*, f32, u8*) { + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_block(const f64*, f64, u8*) { + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_blocks(const f32*, f32, u8*, u32) { + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_blocks(const f64*, f64, u8*, u32) { + static_assert(internal::sve2_compression_unimplemented_v, + "ARM64 SVE2 compression is not implemented yet"); + return -1; +} } // namespace pernix #endif // PERNIX_ARM64_SVE2_COMPRESSION_H diff --git a/include/pernix/fallback/scalar_compression.h b/include/pernix/fallback/scalar_compression.h index fe8d4f3..18e17cb 100644 --- a/include/pernix/fallback/scalar_compression.h +++ b/include/pernix/fallback/scalar_compression.h @@ -1,8 +1,6 @@ #ifndef PERNIX_FALLBACK_SCALAR_COMPRESSION_H #define PERNIX_FALLBACK_SCALAR_COMPRESSION_H -#include - #include #include #include @@ -11,173 +9,173 @@ #include namespace pernix { - namespace internal { +namespace internal { __always_inline i32 quantize_ps_epi32(const float input, const float scale) { - return static_cast(std::lroundf(input * scale)); - } + return static_cast(std::lroundf(input * scale)); +} __always_inline i64 quantize_pd_epi64(const f64 input, const f64 scale) { - return std::llround(input * scale); - } + return std::llround(input * scale); +} - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) __always_inline i32 quantize_clamped(const T input, const T scale) { - constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); - constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); - - const long double scaled = static_cast(input) * static_cast(scale); - if (std::isnan(scaled)) { - return 0; - } - if (scaled <= static_cast(min_value)) { - return static_cast(min_value); - } - if (scaled >= static_cast(max_value)) { - return static_cast(max_value); - } - - return static_cast(std::llround(scaled)); - } + constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); + constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); + + const long double scaled = static_cast(input) * static_cast(scale); + if (std::isnan(scaled)) { + return 0; + } + if (scaled <= static_cast(min_value)) { + return static_cast(min_value); + } + if (scaled >= static_cast(max_value)) { + return static_cast(max_value); + } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) + return static_cast(std::llround(scaled)); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __always_inline i32 clamp_signed_quantized(const i64 value) { - if constexpr (BIT_WIDTH == 1) { - return static_cast(std::clamp(value, 0, 1)); - } + if constexpr (BIT_WIDTH == 1) { + return static_cast(std::clamp(value, 0, 1)); + } - constexpr i32 min_value = -(1 << (BIT_WIDTH - 1)); - constexpr i32 max_value = (1 << (BIT_WIDTH - 1)) - 1; - return static_cast(std::clamp(value, min_value, max_value)); - } + constexpr i32 min_value = -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = (1 << (BIT_WIDTH - 1)) - 1; + return static_cast(std::clamp(value, min_value, max_value)); +} - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) - void pack_epi32_fallback_inner(const std::vector &input, const u8 bit_offset, - u8 * __restrict__ destination) { - constexpr u32 bits_in_type = sizeof(T) * 8; - constexpr u32 bitmask = BIT_WIDTH == bits_in_type - ? std::numeric_limits::max() - : (1U << BIT_WIDTH) - 1U; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) +void pack_epi32_fallback_inner(const std::vector& input, const u8 bit_offset, + u8* __restrict__ destination) { + constexpr u32 bits_in_type = sizeof(T) * 8; + constexpr u32 bitmask = BIT_WIDTH == bits_in_type + ? std::numeric_limits::max() + : (1U << BIT_WIDTH) - 1U; - std::size_t idx = 0; - std::size_t bits_in_buffer = bit_offset; - u64 buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; + std::size_t idx = 0; + std::size_t bits_in_buffer = bit_offset; + u64 buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; #pragma GCC unroll 64 - for (u32 raw_value: input) { - const u32 next_value = raw_value & bitmask; - - buffer |= static_cast(next_value) << bits_in_buffer; - bits_in_buffer += BIT_WIDTH; - - while (bits_in_buffer >= 8) { - destination[idx++] = static_cast(buffer & 0xFFU); - buffer >>= 8; - bits_in_buffer -= 8; - } - } - - if (bits_in_buffer > 0) { - destination[idx] = static_cast(buffer & 0xFFU); - } - } + for (u32 raw_value : input) { + const u32 next_value = raw_value & bitmask; + + buffer |= static_cast(next_value) << bits_in_buffer; + bits_in_buffer += BIT_WIDTH; - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - void pack_epi32_fallback(const std::vector &input, u8 * __restrict__ destination) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } else { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } + while (bits_in_buffer >= 8) { + destination[idx++] = static_cast(buffer & 0xFFU); + buffer >>= 8; + bits_in_buffer -= 8; } } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - int compress_block_fallback(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); + if (bits_in_buffer > 0) { + destination[idx] = static_cast(buffer & 0xFFU); + } +} - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +void pack_epi32_fallback(const std::vector& input, u8* __restrict__ destination) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::pack_epi32_fallback_inner(input, 0, destination); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::pack_epi32_fallback_inner(input, 0, destination); + } else { + return internal::pack_epi32_fallback_inner(input, 0, destination); + } +} +} - std::memset(output, 0, BLOCK_SIZE); +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +int compress_block_fallback(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - std::vector block_values(elements_per_block); -#pragma GCC unroll 64 - for (u32 i = 0; i < elements_per_block; i++) { - const i32 quantized = internal::quantize_clamped(input[i], scale); - block_values[i] = static_cast(quantized); - } + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - internal::pack_epi32_fallback(block_values, output); - return 0; + std::memset(output, 0, BLOCK_SIZE); + + std::vector block_values(elements_per_block); +#pragma GCC unroll 64 + for (u32 i = 0; i < elements_per_block; i++) { + const i32 quantized = internal::quantize_clamped(input[i], scale); + block_values[i] = static_cast(quantized); } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - int compress_block_fallback(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); + internal::pack_epi32_fallback(block_values, output); + return 0; +} - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +int compress_block_fallback(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - std::memset(output, 0, BLOCK_SIZE); + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - std::vector block_values(elements_per_block); -#pragma GCC unroll 32 - for (u32 i = 0; i < elements_per_block; i++) { - const i32 quantized = internal::quantize_clamped(input[i], scale); - block_values[i] = static_cast(quantized); - } + std::memset(output, 0, BLOCK_SIZE); - internal::pack_epi32_fallback(block_values, output); - return 0; + std::vector block_values(elements_per_block); +#pragma GCC unroll 32 + for (u32 i = 0; i < elements_per_block; i++) { + const i32 quantized = internal::quantize_clamped(input[i], scale); + block_values[i] = static_cast(quantized); } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - int compress_blocks_fallback(const void * __restrict__ input_ptr, float scale, void * __restrict__ output_ptr, - u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); + internal::pack_epi32_fallback(block_values, output); + return 0; +} - const f32 *block_input = input; - u8 *block_output = output; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +int compress_blocks_fallback(const void* __restrict__ input_ptr, float scale, void* __restrict__ output_ptr, + u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - for (u32 block = 0; block < blocks; block++) { - compress_block_fallback(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + const f32* block_input = input; + u8* block_output = output; - return 0; + for (u32 block = 0; block < blocks; block++) { + compress_block_fallback(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - int compress_blocks_fallback(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr, - const unsigned int blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const f64 *block_input = input; - u8 *block_output = output; - - for (u32 block = 0; block < blocks; block++) { - compress_block_fallback(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - return 0; + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +int compress_blocks_fallback(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr, + const unsigned int blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f64* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + compress_block_fallback(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; } + return 0; +} } #endif // PERNIX_FALLBACK_SCALAR_COMPRESSION_H diff --git a/include/pernix/fallback/scalar_decompression.h b/include/pernix/fallback/scalar_decompression.h index 720d98a..50556b4 100644 --- a/include/pernix/fallback/scalar_decompression.h +++ b/include/pernix/fallback/scalar_decompression.h @@ -1,165 +1,162 @@ #ifndef PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H #define PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H -#include - -#include #include #include #include namespace pernix { - namespace internal { +namespace internal { __always_inline float dequantize_epi32(const i32 input, const float scale) { - return static_cast(input) * scale; - } + return static_cast(input) * scale; +} __always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { - return static_cast(input) * scale; - } + return static_cast(input) * scale; +} - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - __always_inline auto sign_extend(const u32 value) -> i32 { - if constexpr (BIT_WIDTH == 1) { - return static_cast(value & 1U); - } - - constexpr u32 sign_bit = u32{1} << (BIT_WIDTH - 1); - constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1; - const u32 masked = value & mask; - return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); - } +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__always_inline auto sign_extend(const u32 value) -> i32 { + if constexpr (BIT_WIDTH == 1) { + return static_cast(value & 1U); + } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) - __always_inline auto unpack_epi32_fallback_inner(const u8 * __restrict__ input, const u8 bit_offset, - const std::size_t elements) - -> std::vector { - constexpr u32 bits_in_type = sizeof(T) * 8; - constexpr u32 bitmask = BIT_WIDTH == bits_in_type - ? std::numeric_limits::max() - : (1U << BIT_WIDTH) - 1U; + constexpr u32 sign_bit = u32{1} << (BIT_WIDTH - 1); + constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1; + const u32 masked = value & mask; + return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) +__always_inline auto unpack_epi32_fallback_inner(const u8* __restrict__ input, const u8 bit_offset, + const std::size_t elements) + -> std::vector { + constexpr u32 bits_in_type = sizeof(T) * 8; + constexpr u32 bitmask = BIT_WIDTH == bits_in_type + ? std::numeric_limits::max() + : (1U << BIT_WIDTH) - 1U; - std::vector output(elements); + std::vector output(elements); - std::size_t idx = 0; - u8 bits_in_buffer = 8 - bit_offset; - u64 buffer = static_cast(input[idx++]) >> bit_offset; + std::size_t idx = 0; + u8 bits_in_buffer = 8 - bit_offset; + u64 buffer = static_cast(input[idx++]) >> bit_offset; #pragma GCC unroll 64 - for (u32 i = 0; i < elements; i++) { - while (BIT_WIDTH > bits_in_buffer) { - const auto next_value = static_cast(input[idx++]) << bits_in_buffer; - buffer |= next_value; - bits_in_buffer += 8; - } - - const u32 raw_value = static_cast(buffer & bitmask); - if constexpr (SIGN_VALUES) { - output[i] = sign_extend(raw_value); - } else { - output[i] = static_cast(raw_value); - } - - buffer >>= BIT_WIDTH; - bits_in_buffer -= BIT_WIDTH; - } - - return output; + for (u32 i = 0; i < elements; i++) { + while (BIT_WIDTH > bits_in_buffer) { + const auto next_value = static_cast(input[idx++]) << bits_in_buffer; + buffer |= next_value; + bits_in_buffer += 8; } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - __always_inline auto unpack_epi32_fallback(const u8 * __restrict__ input, - const std::size_t elements) -> std::vector { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else { - return unpack_epi32_fallback_inner(input, 0, elements); - } + const u32 raw_value = static_cast(buffer & bitmask); + if constexpr (SIGN_VALUES) { + output[i] = sign_extend(raw_value); + } else { + output[i] = static_cast(raw_value); } + + buffer >>= BIT_WIDTH; + bits_in_buffer -= BIT_WIDTH; + } + + return output; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__always_inline auto unpack_epi32_fallback(const u8* __restrict__ input, + const std::size_t elements) -> std::vector { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return unpack_epi32_fallback_inner(input, 0, elements); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return unpack_epi32_fallback_inner(input, 0, elements); + } else { + return unpack_epi32_fallback_inner(input, 0, elements); } +} +} - template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) - int decompress_block_fallback(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +int decompress_block_fallback(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const std::vector block_values = internal::unpack_epi32_fallback( - input, elements_per_block); + const std::vector block_values = internal::unpack_epi32_fallback( + input, elements_per_block); #pragma GCC unroll 512 - for (u32 i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi32(block_values[i], scale); - } - - return 0; + for (u32 i = 0; i < elements_per_block; i++) { + output[i] = internal::dequantize_epi32(block_values[i], scale); } - template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) - int decompress_block_fallback(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); + return 0; +} - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +int decompress_block_fallback(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - const std::vector block_values = internal::unpack_epi32_fallback( - input, elements_per_block); + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; -#pragma GCC unroll 512 - for (u32 i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi64(block_values[i], scale); - } + const std::vector block_values = internal::unpack_epi32_fallback( + input, elements_per_block); - return 0; +#pragma GCC unroll 512 + for (u32 i = 0; i < elements_per_block; i++) { + output[i] = internal::dequantize_epi64(block_values[i], scale); } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - int decompress_blocks_fallback(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr, const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); + return 0; +} - const u8 *block_input = input; - f32 *block_output = output; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +int decompress_blocks_fallback(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - for (u32 block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + const u8* block_input = input; + f32* block_output = output; - return 0; + for (u32 block = 0; block < blocks; block++) { + decompress_block_fallback(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } - template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) - int decompress_blocks_fallback(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr, const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); + return 0; +} - const u8 *block_input = input; - f64 *block_output = output; +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +int decompress_blocks_fallback(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - for (u32 block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + const u8* block_input = input; + f64* block_output = output; - return 0; + for (u32 block = 0; block < blocks; block++) { + decompress_block_fallback(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } + + return 0; +} } #endif // PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H diff --git a/include/pernix/x86/avx2/avx2_compression.h b/include/pernix/x86/avx2/avx2_compression.h index b508e0c..4e611d7 100644 --- a/include/pernix/x86/avx2/avx2_compression.h +++ b/include/pernix/x86/avx2/avx2_compression.h @@ -11,68 +11,69 @@ #include namespace pernix { - namespace internal { - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +namespace internal { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __always_inline __m256i mm256_clamp_signed_epi32(__m256i input) { - constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); - constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), - _mm256_set1_epi32(max_value)); - } + constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); + return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), + _mm256_set1_epi32(max_value)); +} + - /** +/** * @brief Quantize four float values into signed 32-bit integers. * * @param input source float lane values. * @param scale per-lane scale factor. * @return __m128i quantized values. */ -__always_inline __m128i mm_quantize_ps_epi32(const __m128 &input, const __m128 &scale) { - const __m128 scaled = _mm_mul_ps(input, scale); - // const __m128 rounded = _mm_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); - return _mm_cvtps_epi32(scaled); - } +__always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& scale) { + const __m128 scaled = _mm_mul_ps(input, scale); + // const __m128 rounded = _mm_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + return _mm_cvtps_epi32(scaled); +} - /** +/** * @brief Quantize two double values into a partially filled 128-bit integer register. * * @param input source double lane values. * @param scale per-lane scale factor. * @return __m128i quantized values in the low lanes. */ -__always_inline __m128i mm_quantize_pd_epi32(const __m128d &input, const __m128d &scale) { - const __m128d scaled = _mm_mul_pd(input, scale); - return _mm_cvtpd_epi32(scaled); - } +__always_inline __m128i mm_quantize_pd_epi32(const __m128d& input, const __m128d& scale) { + const __m128d scaled = _mm_mul_pd(input, scale); + return _mm_cvtpd_epi32(scaled); +} - /** +/** * @brief Quantize eight float values into signed 32-bit integers. * * @param input source float lane values. * @param scale per-lane scale factor. * @return __m256i quantized values. */ -__always_inline __m256i mm256_quantize_ps_epi32(const __m256 &input, const __m256 &scale) { - const __m256 scaled = _mm256_mul_ps(input, scale); - // const __m256 rounded = _mm256_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); - return _mm256_cvtps_epi32(scaled); - } +__always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m256& scale) { + const __m256 scaled = _mm256_mul_ps(input, scale); + // const __m256 rounded = _mm256_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + return _mm256_cvtps_epi32(scaled); +} - /** +/** * @brief Quantize four double values into signed 32-bit integers. * * @param input source double lane values. * @param scale per-lane scale factor. * @return __m128i quantized values in the low lanes. */ -__always_inline __m128i mm256_quantize_pd_epi32(const __m256d &input, const __m256d &scale) { - const __m256d scaled = _mm256_mul_pd(input, scale); - return _mm256_cvtpd_epi32(scaled); - } +__always_inline __m128i mm256_quantize_pd_epi32(const __m256d& input, const __m256d& scale) { + const __m256d scaled = _mm256_mul_pd(input, scale); + return _mm256_cvtpd_epi32(scaled); +} #ifndef PERNIX_USE_SIMDE - /** +/** * @brief Emulate per-16-bit left shifts on AVX2. * * @param a source values. @@ -80,350 +81,350 @@ __always_inline __m128i mm256_quantize_pd_epi32(const __m256d &input, const __m2 * @return __m128i shifted values. */ __always_inline static __m128i _mm_sllv_epi16(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi32(0xffff0000); - const __m128i low_half = _mm_sllv_epi32(a, _mm_andnot_si128(mask, count)); - const __m128i high_half = _mm_sllv_epi32(_mm_and_si128(mask, a), _mm_srli_epi32(count, 16)); - return _mm_blend_epi16(low_half, high_half, 0xaa); - } + const __m128i mask = _mm_set1_epi32(0xffff0000); + const __m128i low_half = _mm_sllv_epi32(a, _mm_andnot_si128(mask, count)); + const __m128i high_half = _mm_sllv_epi32(_mm_and_si128(mask, a), _mm_srli_epi32(count, 16)); + return _mm_blend_epi16(low_half, high_half, 0xaa); +} - /** +/** * @brief Emulate per-16-bit right shifts on AVX2. */ __always_inline static __m128i _mm_srlv_epi16(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi32(0x0000ffff); - const __m128i low_half = _mm_srlv_epi32(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); - const __m128i high_half = _mm_srlv_epi32(a, _mm_srli_epi32(count, 16)); - return _mm_blend_epi16(low_half, high_half, 0xaa); - } + const __m128i mask = _mm_set1_epi32(0x0000ffff); + const __m128i low_half = _mm_srlv_epi32(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); + const __m128i high_half = _mm_srlv_epi32(a, _mm_srli_epi32(count, 16)); + return _mm_blend_epi16(low_half, high_half, 0xaa); +} - /** +/** * @brief Emulate per-16-bit left shifts on 256-bit AVX2 vectors. */ __always_inline static __m256i _mm256_sllv_epi16(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi32(0xffff0000); - const __m256i low_half = _mm256_sllv_epi32(a, _mm256_andnot_si256(mask, count)); - const __m256i high_half = _mm256_sllv_epi32(_mm256_and_si256(mask, a), _mm256_srli_epi32(count, 16)); - return _mm256_blend_epi16(low_half, high_half, 0xaa); - } + const __m256i mask = _mm256_set1_epi32(0xffff0000); + const __m256i low_half = _mm256_sllv_epi32(a, _mm256_andnot_si256(mask, count)); + const __m256i high_half = _mm256_sllv_epi32(_mm256_and_si256(mask, a), _mm256_srli_epi32(count, 16)); + return _mm256_blend_epi16(low_half, high_half, 0xaa); +} - /** +/** * @brief Emulate per-16-bit right shifts on 256-bit AVX2 vectors. */ __always_inline static __m256i _mm256_srlv_epi16(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi32(0x0000ffff); - const __m256i low_half = _mm256_srlv_epi32(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); - const __m256i high_half = _mm256_srlv_epi32(a, _mm256_srli_epi32(count, 16)); - return _mm256_blend_epi16(low_half, high_half, 0xaa); - } + const __m256i mask = _mm256_set1_epi32(0x0000ffff); + const __m256i low_half = _mm256_srlv_epi32(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); + const __m256i high_half = _mm256_srlv_epi32(a, _mm256_srli_epi32(count, 16)); + return _mm256_blend_epi16(low_half, high_half, 0xaa); +} - /** +/** * @brief Blend 8-bit lanes by expanding a scalar mask value. */ __always_inline static __m128i mm_blend_epi8(const __m128i X, const __m128i Y, const i8 M) { - return _mm_blendv_epi8(X, Y, _mm_set1_epi8(M)); - } + return _mm_blendv_epi8(X, Y, _mm_set1_epi8(M)); +} - /** +/** * @brief Blend 8-bit lanes in 256-bit vectors by expanding a scalar mask value. */ __always_inline static __m256i mm256_blend_epi8(const __m256i X, const __m256i Y, const i8 M) { - return _mm256_blendv_epi8(X, Y, _mm256_set1_epi8(M)); - } + return _mm256_blendv_epi8(X, Y, _mm256_set1_epi8(M)); +} - /** +/** * @brief Emulate per-byte left shifts on 128-bit vectors. */ __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0xff00); - const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); - const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); - return mm_blend_epi8(low_half, high_half, 0xaa); - } + const __m128i mask = _mm_set1_epi16(0xff00); + const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); + const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); + return mm_blend_epi8(low_half, high_half, 0xaa); +} - /** +/** * @brief Emulate per-byte right shifts on 128-bit vectors. */ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0x00ff); - const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); - const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); - return mm_blend_epi8(low_half, high_half, 0xaa); - } + const __m128i mask = _mm_set1_epi16(0x00ff); + const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); + const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); + return mm_blend_epi8(low_half, high_half, 0xaa); +} - /** +/** * @brief Emulate per-byte left shifts on 256-bit vectors. */ __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0xff00); - const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); - const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); - return mm256_blend_epi8(low_half, high_half, 0xaa); - } + const __m256i mask = _mm256_set1_epi16(0xff00); + const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); + const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); + return mm256_blend_epi8(low_half, high_half, 0xaa); +} - /** +/** * @brief Emulate per-byte right shifts on 256-bit vectors. */ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0x00ff); - const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); - const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); - return mm256_blend_epi8(low_half, high_half, 0xaa); - } + const __m256i mask = _mm256_set1_epi16(0x00ff); + const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); + const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); + return mm256_blend_epi8(low_half, high_half, 0xaa); +} #endif - /** +/** * @brief Pack four 32-bit values for bit widths 1 through 3. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) - __always_inline auto mm_pack_epi32_avx2_1to3(__m128i &input) -> __m128i { - constexpr u32 bitmask = (1U << BIT_WIDTH) - 1U; - const __m128i masked = _mm_and_si128(input, _mm_set1_epi32(static_cast(bitmask))); +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) +__always_inline auto mm_pack_epi32_avx2_1to3(__m128i& input) -> __m128i { + constexpr u32 bitmask = (1U << BIT_WIDTH) - 1U; + const __m128i masked = _mm_and_si128(input, _mm_set1_epi32(static_cast(bitmask))); - alignas(16) u32 lanes[4]; - _mm_storeu_si128(reinterpret_cast<__m128i *>(lanes), masked); + alignas(16) u32 lanes[4]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(lanes), masked); - const u32 packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ( - (lanes[2] & bitmask) << (2 * BIT_WIDTH)) | - ((lanes[3] & bitmask) << (3 * BIT_WIDTH)); + const u32 packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ( + (lanes[2] & bitmask) << (2 * BIT_WIDTH)) | + ((lanes[3] & bitmask) << (3 * BIT_WIDTH)); - return _mm_cvtsi32_si128(static_cast(packed)); - } + return _mm_cvtsi32_si128(static_cast(packed)); +} - /** +/** * @brief Pack eight 32-bit values for bit widths 1 through 3. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) -__always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i &input) { - constexpr u32 bitmask = (1u << BIT_WIDTH) - 1u; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) +__always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { + constexpr u32 bitmask = (1u << BIT_WIDTH) - 1u; - const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(static_cast(bitmask))); + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(static_cast(bitmask))); - const __m256i shifts = _mm256_setr_epi32(0 * BIT_WIDTH, 1 * BIT_WIDTH, 2 * BIT_WIDTH, 3 * BIT_WIDTH, - 4 * BIT_WIDTH, 5 * BIT_WIDTH, - 6 * BIT_WIDTH, 7 * BIT_WIDTH); + const __m256i shifts = _mm256_setr_epi32(0 * BIT_WIDTH, 1 * BIT_WIDTH, 2 * BIT_WIDTH, 3 * BIT_WIDTH, + 4 * BIT_WIDTH, 5 * BIT_WIDTH, + 6 * BIT_WIDTH, 7 * BIT_WIDTH); - const __m256i shifted = _mm256_sllv_epi32(masked, shifts); + const __m256i shifted = _mm256_sllv_epi32(masked, shifts); - __m128i x = _mm_or_si128(_mm256_castsi256_si128(shifted), _mm256_extracti128_si256(shifted, 1)); + __m128i x = _mm_or_si128(_mm256_castsi256_si128(shifted), _mm256_extracti128_si256(shifted, 1)); - x = _mm_or_si128(x, _mm_srli_si128(x, 8)); - x = _mm_or_si128(x, _mm_srli_si128(x, 4)); + x = _mm_or_si128(x, _mm_srli_si128(x, 8)); + x = _mm_or_si128(x, _mm_srli_si128(x, 4)); - return _mm256_castsi128_si256(x); - } + return _mm256_castsi128_si256(x); +} -__always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i &input) { - const __m256i zero = _mm256_setzero_si256(); +__always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i& input) { + const __m256i zero = _mm256_setzero_si256(); - const __m256i packed16 = _mm256_packus_epi32(input, zero); - const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); - const __m256i packed8 = _mm256_packus_epi16(permuted, zero); + const __m256i packed16 = _mm256_packus_epi32(input, zero); + const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); + const __m256i packed8 = _mm256_packus_epi16(permuted, zero); - const __m256i combined = _mm256_or_si256(packed8, _mm256_srli_epi16(packed8, 4)); + const __m256i combined = _mm256_or_si256(packed8, _mm256_srli_epi16(packed8, 4)); - const __m256i shuffled = _mm256_shuffle_epi8(combined, _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 2, - 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i shuffled = _mm256_shuffle_epi8(combined, _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 2, + 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1)); - return shuffled; - } + return shuffled; +} - /** +/** * @brief Pack four 32-bit values for bit widths 9 through 16. */ - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) - __always_inline auto mm_pack_epi32_avx2_9to16(__m128i &input) -> __m128i { - using tables = pack_tables_avx2_16; - constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; - const __m128i masked = _mm_and_si128(input, _mm_set1_epi16(bitmask)); - __m128i combined; - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); - const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); - - const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); - const __m128i shifted2 = _mm_srlv_epi16(shuffled2, tables::get_shift2()); - - combined = _mm_or_si128(shifted1, shifted2); - } else { - const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); - const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); - const __m128i shuffled3 = _mm_shuffle_epi8(masked, tables::get_permute3()); - - const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi16(shuffled2, tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi16(shuffled3, tables::get_shift3()); - - combined = _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); - } - return combined; - } +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { + using tables = pack_tables_avx2_16; + constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; + const __m128i masked = _mm_and_si128(input, _mm_set1_epi16(bitmask)); + __m128i combined; + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); + const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); + + const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); + const __m128i shifted2 = _mm_srlv_epi16(shuffled2, tables::get_shift2()); + + combined = _mm_or_si128(shifted1, shifted2); + } else { + const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); + const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); + const __m128i shuffled3 = _mm_shuffle_epi8(masked, tables::get_permute3()); + + const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi16(shuffled2, tables::get_shift2()); + const __m128i shifted3 = _mm_srlv_epi16(shuffled3, tables::get_shift3()); + + combined = _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); + } + return combined; +} - /** +/** * @brief Pack eight 32-bit values for bit widths 9 through 16. */ - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) - __always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i &input) -> __m256i { - using tables = pack_tables_avx2_16; - constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; - const __m128i packed = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); - const __m128i masked = _mm_and_si128(packed, _mm_set1_epi16(bitmask)); - __m128i combined; - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15 || BIT_WIDTH == 16) { - const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); - const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); - - const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); - const __m128i shifted2 = _mm_srlv_epi16(shuffled2, tables::get_shift2()); - - combined = _mm_or_si128(shifted1, shifted2); - } else { - const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); - const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); - const __m128i shuffled3 = _mm_shuffle_epi8(masked, tables::get_permute3()); - - const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi16(shuffled2, tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi16(shuffled3, tables::get_shift3()); - - combined = _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); - } - return _mm256_castsi128_si256(combined); - } +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i& input) -> __m256i { + using tables = pack_tables_avx2_16; + constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; + const __m128i packed = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); + const __m128i masked = _mm_and_si128(packed, _mm_set1_epi16(bitmask)); + __m128i combined; + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15 || BIT_WIDTH == 16) { + const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); + const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); + + const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); + const __m128i shifted2 = _mm_srlv_epi16(shuffled2, tables::get_shift2()); + + combined = _mm_or_si128(shifted1, shifted2); + } else { + const __m128i shuffled1 = _mm_shuffle_epi8(masked, tables::get_permute1()); + const __m128i shuffled2 = _mm_shuffle_epi8(masked, tables::get_permute2()); + const __m128i shuffled3 = _mm_shuffle_epi8(masked, tables::get_permute3()); + + const __m128i shifted1 = _mm_sllv_epi16(shuffled1, tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi16(shuffled2, tables::get_shift2()); + const __m128i shifted3 = _mm_srlv_epi16(shuffled3, tables::get_shift3()); + + combined = _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); + } + return _mm256_castsi128_si256(combined); +} - template - requires(BIT_WIDTH >= 5 && BIT_WIDTH <= 7) - __always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i &input) -> __m256i { - const __m256i zero = _mm256_setzero_si256(); +template + requires(BIT_WIDTH >= 5 && BIT_WIDTH <= 7) +__always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i { + const __m256i zero = _mm256_setzero_si256(); - const __m256i packed16 = _mm256_packus_epi32(input, zero); - const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); - const __m256i packed8 = _mm256_packus_epi16(permuted, zero); + const __m256i packed16 = _mm256_packus_epi32(input, zero); + const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); + const __m256i packed8 = _mm256_packus_epi16(permuted, zero); - const __m256i even = _mm256_and_si256(packed8, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(packed8, _mm256_set1_epi16(0xFF00)); + const __m256i even = _mm256_and_si256(packed8, _mm256_set1_epi16(0x00FF)); + const __m256i odd = _mm256_and_si256(packed8, _mm256_set1_epi16(0xFF00)); - const __m256i pair16 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); - const __m256i extended = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(pair16)); + const __m256i pair16 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); + const __m256i extended = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(pair16)); - return mm256_pack_epi32_avx2_9to16<2 * BIT_WIDTH>(extended); - } + return mm256_pack_epi32_avx2_9to16<2 * BIT_WIDTH>(extended); +} - /** +/** * @brief Pack eight 32-bit values for bit widths 17 through 24. */ - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) - __always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i &input) -> __m256i { - using tables = pack_tables_avx2_24; - constexpr u32 bitmask = (1 << BIT_WIDTH) - 1; - const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(bitmask)); - __m256i combined; - - if constexpr (BIT_WIDTH == 24) { - const __m256i shuffled1 = _mm256_permutevar8x32_epi32(masked, tables::get_permute1()); - const __m256i shuffled2 = _mm256_permutevar8x32_epi32(masked, tables::get_permute2()); - - const __m256i shifted1 = _mm256_sllv_epi32(shuffled1, tables::get_shift1()); - const __m256i shifted2 = _mm256_srlv_epi32(shuffled2, tables::get_shift2()); - - combined = _mm256_or_si256(shifted1, shifted2); - } else { - const __m256i shuffled1 = _mm256_permutevar8x32_epi32(masked, tables::get_permute1()); - const __m256i shuffled2 = _mm256_permutevar8x32_epi32(masked, tables::get_permute2()); - const __m256i shuffled3 = _mm256_permutevar8x32_epi32(masked, tables::get_permute3()); - - const __m256i shifted1 = _mm256_sllv_epi32(shuffled1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi32(shuffled2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi32(shuffled3, tables::get_shift3()); - - combined = _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); - } - - return combined; - } +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m256i { + using tables = pack_tables_avx2_24; + constexpr u32 bitmask = (1 << BIT_WIDTH) - 1; + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(bitmask)); + __m256i combined; + + if constexpr (BIT_WIDTH == 24) { + const __m256i shuffled1 = _mm256_permutevar8x32_epi32(masked, tables::get_permute1()); + const __m256i shuffled2 = _mm256_permutevar8x32_epi32(masked, tables::get_permute2()); + + const __m256i shifted1 = _mm256_sllv_epi32(shuffled1, tables::get_shift1()); + const __m256i shifted2 = _mm256_srlv_epi32(shuffled2, tables::get_shift2()); + + combined = _mm256_or_si256(shifted1, shifted2); + } else { + const __m256i shuffled1 = _mm256_permutevar8x32_epi32(masked, tables::get_permute1()); + const __m256i shuffled2 = _mm256_permutevar8x32_epi32(masked, tables::get_permute2()); + const __m256i shuffled3 = _mm256_permutevar8x32_epi32(masked, tables::get_permute3()); + + const __m256i shifted1 = _mm256_sllv_epi32(shuffled1, tables::get_shift1()); + const __m256i shifted2 = _mm256_sllv_epi32(shuffled2, tables::get_shift2()); + const __m256i shifted3 = _mm256_srlv_epi32(shuffled3, tables::get_shift3()); + + combined = _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); + } + + return combined; +} - /** +/** * @brief Pack aligned 8-bit or 16-bit values from four 32-bit lanes. */ - template - requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) - auto mm_pack_aligned_epi32_avx2(__m128i &input) -> __m128i { - if constexpr (BIT_WIDTH == 8) { - return _mm_packus_epi16(_mm_packs_epi32(input, _mm_setzero_si128()), _mm_setzero_si128()); - } else { - return _mm_packs_epi32(input, _mm_setzero_si128()); - } - } +template + requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) +auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { + if constexpr (BIT_WIDTH == 8) { + return _mm_packus_epi16(_mm_packs_epi32(input, _mm_setzero_si128()), _mm_setzero_si128()); + } else { + return _mm_packs_epi32(input, _mm_setzero_si128()); + } +} - /** +/** * @brief Dispatch to the appropriate 128-bit AVX2 packer for the selected bit width. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 16) - auto mm_pack_epi32_avx2(__m128i &input) -> __m128i { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { - return internal::mm_pack_epi32_avx2_1to3(input); - } else if constexpr (BIT_WIDTH >= 4 && BIT_WIDTH <= 8) { - // TODO: implementation for 4-8 bits - return _mm_setzero_si128(); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::mm_pack_epi32_avx2_9to16(input); - } else { - return _mm_setzero_si128(); - } - } +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 16) +auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { + return internal::mm_pack_epi32_avx2_1to3(input); + } else if constexpr (BIT_WIDTH >= 4 && BIT_WIDTH <= 8) { + // TODO: implementation for 4-8 bits + return _mm_setzero_si128(); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::mm_pack_epi32_avx2_9to16(input); + } else { + return _mm_setzero_si128(); + } +} - /** +/** * @brief Pack aligned 8-bit or 16-bit values from eight 32-bit lanes. */ - template - requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) - __m256i mm256_pack_aligned_epi32_avx2(const __m256i &input) { - if constexpr (BIT_WIDTH == 8) { - const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), - _mm256_extracti128_si256(input, 1)); - const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); - return _mm256_castsi128_si256(packed8); - } else { - return _mm256_castsi128_si256( - _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1))); - } - } +template + requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) +__m256i mm256_pack_aligned_epi32_avx2(const __m256i& input) { + if constexpr (BIT_WIDTH == 8) { + const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), + _mm256_extracti128_si256(input, 1)); + const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); + return _mm256_castsi128_si256(packed8); + } else { + return _mm256_castsi128_si256( + _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1))); + } +} - /** +/** * @brief Dispatch to the appropriate 256-bit AVX2 packer for the selected bit width. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - __m256i mm256_pack_epi32_avx2(const __m256i &input) { - if constexpr (BIT_WIDTH == 8 || BIT_WIDTH == 16) { - return internal::mm256_pack_aligned_epi32_avx2(input); - } else { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { - return internal::mm256_pack_epi32_avx2_1to3(input); - } else if constexpr (BIT_WIDTH == 4) { - return mm256_pack_epi32_avx2_4(input); - } else if constexpr (BIT_WIDTH >= 5 && BIT_WIDTH <= 7) { - return internal::mm256_pack_epi32_avx2_5to7(input); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 15) { - return internal::mm256_pack_epi32_avx2_9to16(input); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::mm256_pack_epi32_avx2_17to24(input); - } - } - return _mm256_setzero_si256(); +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__m256i mm256_pack_epi32_avx2(const __m256i& input) { + if constexpr (BIT_WIDTH == 8 || BIT_WIDTH == 16) { + return internal::mm256_pack_aligned_epi32_avx2(input); + } else { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { + return internal::mm256_pack_epi32_avx2_1to3(input); + } else if constexpr (BIT_WIDTH == 4) { + return mm256_pack_epi32_avx2_4(input); + } else if constexpr (BIT_WIDTH >= 5 && BIT_WIDTH <= 7) { + return internal::mm256_pack_epi32_avx2_5to7(input); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 15) { + return internal::mm256_pack_epi32_avx2_9to16(input); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::mm256_pack_epi32_avx2_17to24(input); } - } // namespace internal + } + return _mm256_setzero_si256(); +} +} // namespace internal - /** +/** * @brief Compress a single block of float using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -436,48 +437,48 @@ __always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i &input) { * * @note This function requires AVX2 support. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_compress_block_avx2(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 iterations_8 = elements_per_block / 8; - constexpr u8 remaining = elements_per_block - iterations_8 * 8; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - const __m256 scale_v = _mm256_set1_ps(scale); + const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 8 - for (u32 iter = 0; iter < iterations_8; iter++) { - const __m256 source = _mm256_loadu_ps(input); - const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); - const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); - const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); - std::memcpy(output, &packed, BIT_WIDTH); - - input += 8; - output += BIT_WIDTH; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256 source = _mm256_loadu_ps(input); + const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + std::memcpy(output, &packed, BIT_WIDTH); + + input += 8; + output += BIT_WIDTH; + } - if constexpr (remaining) { - std::vector block_values(remaining); + if constexpr (remaining) { + std::vector block_values(remaining); #pragma GCC unroll 8 - for (u32 i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized < BIT_WIDTH > ( - internal::quantize_ps_epi32(input[i], scale))); - } - - internal::pack_epi32_fallback < BIT_WIDTH > (block_values, output); + for (u32 i = 0; i < remaining; i++) { + block_values[i] = + static_cast(internal::clamp_signed_quantized( + internal::quantize_ps_epi32(input[i], scale))); } - return 0; + internal::pack_epi32_fallback(block_values, output); } - /** + return 0; +} + +/** * @brief Compress a single block of double using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -490,52 +491,52 @@ __always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i &input) { * * @note This function requires AVX2 support. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_compress_block_avx2(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 iterations_8 = elements_per_block / 8; - constexpr u8 remaining = elements_per_block - iterations_8 * 8; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - const __m256d scale_v = _mm256_set1_pd(scale); + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 8 - for (u32 iter = 0; iter < iterations_8; iter++) { - const __m256d source1 = _mm256_loadu_pd(input); - const __m256d source2 = _mm256_loadu_pd(input + 4); - const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); - const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); - __m256i combined = _mm256_castsi128_si256(quantized1); - combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_avx2( - internal::mm256_clamp_signed_epi32(combined)); - // _mm_storeu_si128(reinterpret_cast<__m128i*>(output), _mm256_castsi256_si128(packed)); - std::memcpy(output, &packed, BIT_WIDTH); - input += 8; - output += BIT_WIDTH; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256d source1 = _mm256_loadu_pd(input); + const __m256d source2 = _mm256_loadu_pd(input + 4); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed = internal::mm256_pack_epi32_avx2( + internal::mm256_clamp_signed_epi32(combined)); + // _mm_storeu_si128(reinterpret_cast<__m128i*>(output), _mm256_castsi256_si128(packed)); + std::memcpy(output, &packed, BIT_WIDTH); + input += 8; + output += BIT_WIDTH; + } - if constexpr (remaining) { - std::vector block_values(remaining); + if constexpr (remaining) { + std::vector block_values(remaining); #pragma GCC unroll 8 - for (u32 i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized < BIT_WIDTH > ( - internal::quantize_pd_epi64(input[i], scale))); - } - - internal::pack_epi32_fallback < BIT_WIDTH > (block_values, output); + for (u32 i = 0; i < remaining; i++) { + block_values[i] = + static_cast(internal::clamp_signed_quantized( + internal::quantize_pd_epi64(input[i], scale))); } - return 0; + internal::pack_epi32_fallback(block_values, output); } - /** + return 0; +} + +/** * @brief Compress multiple blocks using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -549,27 +550,27 @@ __always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i &input) { * * @note This function requires AVX2 support. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_compress_blocks_avx2(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const f32 *block_input = input; - u8 *block_output = output; - - for (u32 block = 0; block < blocks; block++) { - mm256_compress_block_avx2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - - return 0; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f32* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_avx2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; } - /** + return 0; +} + +/** * @brief Compress multiple blocks using AVX2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -583,25 +584,25 @@ __always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i &input) { * * @note This function requires AVX2 support. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_compress_blocks_avx2(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const f64 *block_input = input; - u8 *block_output = output; - - for (u32 block = 0; block < blocks; block++) { - mm256_compress_block_avx2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - - return 0; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f64* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_avx2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; } + + return 0; +} } // namespace pernix #endif // PERNIX_AVX2_COMPRESSION_H diff --git a/include/pernix/x86/avx2/avx2_decompression.h b/include/pernix/x86/avx2/avx2_decompression.h index 93f2edd..f5af160 100644 --- a/include/pernix/x86/avx2/avx2_decompression.h +++ b/include/pernix/x86/avx2/avx2_decompression.h @@ -11,339 +11,340 @@ #include namespace pernix { - namespace internal { - /** - * @brief Convert an 8-lane mask to the lane representation used by AVX2 float masked stores. - */ +namespace internal { +/** +* @brief Convert an 8-lane mask to the lane representation used by AVX2 float masked stores. +*/ __always_inline __m256i mm256_convert_vmask_epi32(const __mmask8 mask8) { - return _mm256_setr_epi32((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, - (mask8 & 0x8) ? -1 : 0, - (mask8 & 0x10) ? -1 : 0, (mask8 & 0x20) ? -1 : 0, (mask8 & 0x40) ? -1 : 0, - (mask8 & 0x80) ? -1 : 0); - } - - /** - * @brief Convert a 4-lane mask to the lane representation used by AVX2 double masked stores. - */ + return _mm256_setr_epi32((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, + (mask8 & 0x8) ? -1 : 0, + (mask8 & 0x10) ? -1 : 0, (mask8 & 0x20) ? -1 : 0, (mask8 & 0x40) ? -1 : 0, + (mask8 & 0x80) ? -1 : 0); +} + +/** +* @brief Convert a 4-lane mask to the lane representation used by AVX2 double masked stores. +*/ __always_inline __m256i mm256_convert_vmask_epi64(const __mmask8 mask8) { - return _mm256_setr_epi64x((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, - (mask8 & 0x8) ? -1 : 0); - } - - /** - * @brief Dequantize four 32-bit integers to floats. - */ -__always_inline __m128 mm_dequantize_epi32(const __m128i &input, const __m128 &scale) { - const __m128 converted = _mm_cvtepi32_ps(input); - return _mm_mul_ps(converted, scale); - } - - /* https://stackoverflow.com/questions/41144668/how-to-efficiently-perform-double-int64-conversions-with-sse-avx */ -__always_inline __m128d convert_epi64_pd(const __m128i v) { - __m128i xH = _mm_srai_epi32(v, 16); - xH = _mm_blend_epi16(xH, _mm_setzero_si128(), 0x33); - xH = _mm_add_epi64(xH, _mm_castpd_si128(_mm_set1_pd(442721857769029238784.))); // 3*2^67 - const __m128i xL = _mm_blend_epi16(v, _mm_castpd_si128(_mm_set1_pd(0x0010000000000000)), 0x88); // 2^52 - const __m128d f = _mm_sub_pd(_mm_castsi128_pd(xH), _mm_set1_pd(442726361368656609280.)); // 3*2^67 + 2^52 - return _mm_add_pd(f, _mm_castsi128_pd(xL)); - } + return _mm256_setr_epi64x((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, + (mask8 & 0x8) ? -1 : 0); +} - /** - * @brief Dequantize two 64-bit integers to doubles. - */ -__always_inline __m128d mm_dequantize_epi64_pd(const __m128i &input, const __m128d &scale) { - const __m128d converted = convert_epi64_pd(input); - return _mm_mul_pd(converted, scale); - } - /** - * @brief Dequantize eight 32-bit integers to floats. - */ -__always_inline __m256 mm256_dequantize_epi32(const __m256i &input, const __m256 &scale) { - const __m256 converted = _mm256_cvtepi32_ps(input); - return _mm256_mul_ps(converted, scale); - } +/** +* @brief Dequantize four 32-bit integers to floats. +*/ +__always_inline __m128 mm_dequantize_epi32(const __m128i& input, const __m128& scale) { + const __m128 converted = _mm_cvtepi32_ps(input); + return _mm_mul_ps(converted, scale); +} - /* https://stackoverflow.com/questions/41144668/how-to-efficiently-perform-double-int64-conversions-with-sse-avx */ +/* https://stackoverflow.com/questions/41144668/how-to-efficiently-perform-double-int64-conversions-with-sse-avx */ +__always_inline __m128d convert_epi64_pd(const __m128i v) { + __m128i xH = _mm_srai_epi32(v, 16); + xH = _mm_blend_epi16(xH, _mm_setzero_si128(), 0x33); + xH = _mm_add_epi64(xH, _mm_castpd_si128(_mm_set1_pd(442721857769029238784.))); // 3*2^67 + const __m128i xL = _mm_blend_epi16(v, _mm_castpd_si128(_mm_set1_pd(0x0010000000000000)), 0x88); // 2^52 + const __m128d f = _mm_sub_pd(_mm_castsi128_pd(xH), _mm_set1_pd(442726361368656609280.)); // 3*2^67 + 2^52 + return _mm_add_pd(f, _mm_castsi128_pd(xL)); +} + +/** +* @brief Dequantize two 64-bit integers to doubles. +*/ +__always_inline __m128d mm_dequantize_epi64_pd(const __m128i& input, const __m128d& scale) { + const __m128d converted = convert_epi64_pd(input); + return _mm_mul_pd(converted, scale); +} + +/** +* @brief Dequantize eight 32-bit integers to floats. +*/ +__always_inline __m256 mm256_dequantize_epi32(const __m256i& input, const __m256& scale) { + const __m256 converted = _mm256_cvtepi32_ps(input); + return _mm256_mul_ps(converted, scale); +} + +/* https://stackoverflow.com/questions/41144668/how-to-efficiently-perform-double-int64-conversions-with-sse-avx */ __always_inline __m256d convert_epi64_pd(__m256i v) { - __m256i xH = _mm256_srai_epi32(v, 16); - xH = _mm256_blend_epi16(xH, _mm256_setzero_si256(), 0x33); - xH = _mm256_add_epi64(xH, _mm256_castpd_si256(_mm256_set1_pd(442721857769029238784.))); - const __m256i xL = _mm256_blend_epi16(v, _mm256_castpd_si256(_mm256_set1_pd(0x0010000000000000)), 0x88); - const __m256d f = _mm256_sub_pd(_mm256_castsi256_pd(xH), _mm256_set1_pd(442726361368656609280.)); - return _mm256_add_pd(f, _mm256_castsi256_pd(xL)); + __m256i xH = _mm256_srai_epi32(v, 16); + xH = _mm256_blend_epi16(xH, _mm256_setzero_si256(), 0x33); + xH = _mm256_add_epi64(xH, _mm256_castpd_si256(_mm256_set1_pd(442721857769029238784.))); + const __m256i xL = _mm256_blend_epi16(v, _mm256_castpd_si256(_mm256_set1_pd(0x0010000000000000)), 0x88); + const __m256d f = _mm256_sub_pd(_mm256_castsi256_pd(xH), _mm256_set1_pd(442726361368656609280.)); + return _mm256_add_pd(f, _mm256_castsi256_pd(xL)); +} + +/** +* @brief Dequantize four 64-bit integers to doubles. +*/ +__always_inline __m256d mm256_dequantize_epi64_pd(const __m256i& input, const __m256d& scale) { + const __m256d converted = convert_epi64_pd(input); + return _mm256_mul_pd(converted, scale); +} + +/** +* @brief Unpack four aligned 8-bit or 16-bit values directly from the input buffer. +*/ +template + requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) +__m128i mm_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { + if constexpr (BIT_WIDTH == 8) { + const __m128i source = _mm_loadu_si32(input); + if constexpr (SIGN_VALUES) { + return _mm_cvtepi8_epi32(source); + } else { + return _mm_cvtepu8_epi32(source); } - - /** - * @brief Dequantize four 64-bit integers to doubles. - */ -__always_inline __m256d mm256_dequantize_epi64_pd(const __m256i &input, const __m256d &scale) { - const __m256d converted = convert_epi64_pd(input); - return _mm256_mul_pd(converted, scale); + } else { + const __m128i source = _mm_loadu_si64(input); + if constexpr (SIGN_VALUES) { + return _mm_cvtepi16_epi32(source); + } else { + return _mm_cvtepu16_epi32(source); } + } +} + +/** +* @brief Unpack four values using the table-driven AVX2 shuffle path. +*/ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +__m128i mm_unpack_epi32_avx2(const u8* __restrict__ input) { + using unpack_table = unpack_tables_avx2; + constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; + + __m128i source = _mm_setzero_si128(); + std::memcpy(&source, input, packed_bytes); + + const __m128i shuffled = _mm_shuffle_epi8(source, unpack_table::get_shuffle()); + + constexpr u16 shift = 32 - BIT_WIDTH; + __m128i shifted = _mm_sllv_epi32(shuffled, unpack_table::get_shift()); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + shifted = _mm_srai_epi32(shifted, shift); + } else { + shifted = _mm_srli_epi32(shifted, shift); + } - /** - * @brief Unpack four aligned 8-bit or 16-bit values directly from the input buffer. - */ - template - requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) - __m128i mm_unpack_aligned_epi32_avx2(const u8 * __restrict__ input) { - if constexpr (BIT_WIDTH == 8) { - const __m128i source = _mm_loadu_si32(input); - if constexpr (SIGN_VALUES) { - return _mm_cvtepi8_epi32(source); - } else { - return _mm_cvtepu8_epi32(source); - } - } else { - const __m128i source = _mm_loadu_si64(input); - if constexpr (SIGN_VALUES) { - return _mm_cvtepi16_epi32(source); - } else { - return _mm_cvtepu16_epi32(source); - } - } + return shifted; +} + +/** +* @brief Unpack eight aligned 8-bit or 16-bit values directly from the input buffer. +*/ +template + requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) +__m256i mm256_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { + if constexpr (BIT_WIDTH == 8) { + const __m128i source = _mm_loadu_si64(input); + if constexpr (SIGN_VALUES) { + return _mm256_cvtepi8_epi32(source); + } else { + return _mm256_cvtepu8_epi32(source); } - - /** - * @brief Unpack four values using the table-driven AVX2 shuffle path. - */ - template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) - __m128i mm_unpack_epi32_avx2(const u8 * __restrict__ input) { - using unpack_table = unpack_tables_avx2; - constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; - - __m128i source = _mm_setzero_si128(); - std::memcpy(&source, input, packed_bytes); - - const __m128i shuffled = _mm_shuffle_epi8(source, unpack_table::get_shuffle()); - - constexpr u16 shift = 32 - BIT_WIDTH; - __m128i shifted = _mm_sllv_epi32(shuffled, unpack_table::get_shift()); - if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { - shifted = _mm_srai_epi32(shifted, shift); - } else { - shifted = _mm_srli_epi32(shifted, shift); - } - - return shifted; - } - - /** - * @brief Unpack eight aligned 8-bit or 16-bit values directly from the input buffer. - */ - template - requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) - __m256i mm256_unpack_aligned_epi32_avx2(const u8 * __restrict__ input) { - if constexpr (BIT_WIDTH == 8) { - const __m128i source = _mm_loadu_si64(input); - if constexpr (SIGN_VALUES) { - return _mm256_cvtepi8_epi32(source); - } else { - return _mm256_cvtepu8_epi32(source); - } - } else { - const __m128i source = _mm_loadu_si128(reinterpret_cast(input)); - if constexpr (SIGN_VALUES) { - return _mm256_cvtepi16_epi32(source); - } else { - return _mm256_cvtepu16_epi32(source); - } - } + } else { + const __m128i source = _mm_loadu_si128(reinterpret_cast(input)); + if constexpr (SIGN_VALUES) { + return _mm256_cvtepi16_epi32(source); + } else { + return _mm256_cvtepu16_epi32(source); } + } +} + +/** +* @brief Unpack eight values using the table-driven AVX2 shuffle path. +*/ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +__m256i mm256_unpack_epi32_avx2(const u8* __restrict__ input) { + using unpack_table = unpack_tables_avx2; + constexpr std::size_t packed_bytes = BIT_WIDTH; + + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, packed_bytes); + + const __m256i permuted = _mm256_permutevar8x32_epi32(source, unpack_table::get_permute()); + const __m256i shuffled = _mm256_shuffle_epi8(permuted, unpack_table::get_shuffle()); + + constexpr u16 shift = 32 - BIT_WIDTH; + __m256i shifted = _mm256_sllv_epi32(shuffled, unpack_table::get_shift()); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + shifted = _mm256_srai_epi32(shifted, shift); + } else { + shifted = _mm256_srli_epi32(shifted, shift); + } - /** - * @brief Unpack eight values using the table-driven AVX2 shuffle path. - */ - template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) - __m256i mm256_unpack_epi32_avx2(const u8 * __restrict__ input) { - using unpack_table = unpack_tables_avx2; - constexpr std::size_t packed_bytes = BIT_WIDTH; - - __m256i source = _mm256_setzero_si256(); - std::memcpy(&source, input, packed_bytes); - - const __m256i permuted = _mm256_permutevar8x32_epi32(source, unpack_table::get_permute()); - const __m256i shuffled = _mm256_shuffle_epi8(permuted, unpack_table::get_shuffle()); - - constexpr u16 shift = 32 - BIT_WIDTH; - __m256i shifted = _mm256_sllv_epi32(shuffled, unpack_table::get_shift()); - if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { - shifted = _mm256_srai_epi32(shifted, shift); - } else { - shifted = _mm256_srli_epi32(shifted, shift); - } - - return shifted; - } - } // namespace internal - - /** - * @brief Decompress a single block to float using AVX2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_decompress_block_avx2(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 iterations_8 = elements_per_block / 8; - constexpr u8 remaining = elements_per_block - iterations_8 * 8; - - const __m256 scale_v = _mm256_set1_ps(scale); + return shifted; +} +} // namespace internal + +/** +* @brief Decompress a single block to float using AVX2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* +* @param input pointer to the start of the compressed block. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed float values will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX2 support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + + const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 4 - for (u32 iter = 0; iter < iterations_8; iter++) { - const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); - const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); - _mm256_storeu_ps(output, dequantized); - input += BIT_WIDTH; - output += 8; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); + const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); + _mm256_storeu_ps(output, dequantized); + input += BIT_WIDTH; + output += 8; + } - if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback < BIT_WIDTH, SIGN_VALUES + if constexpr (remaining > 0) { + const std::vector tail_values = internal::unpack_epi32_fallback (input, remaining); - for (u32 i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi32(tail_values[i], scale); - } + for (u32 i = 0; i < remaining; i++) { + output[i] = internal::dequantize_epi32(tail_values[i], scale); } - - return 0; } - /** - * @brief Decompress a single block to double using AVX2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_decompress_block_avx2(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 iterations_8 = elements_per_block / 8; - constexpr u8 remaining = elements_per_block - iterations_8 * 8; - const __m256d scale_v = _mm256_set1_pd(scale); + return 0; +} + +/** +* @brief Decompress a single block to double using AVX2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* +* @param input pointer to the start of the compressed block. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed double values will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX2 support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 4 - for (u32 iter = 0; iter < iterations_8; iter++) { - const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); - const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); - const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); + const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); + const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); - const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); - const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); + const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); + const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); - _mm256_storeu_pd(output, dequantized1); - _mm256_storeu_pd(output + 4, dequantized2); + _mm256_storeu_pd(output, dequantized1); + _mm256_storeu_pd(output + 4, dequantized2); - input += BIT_WIDTH; - output += 8; - } + input += BIT_WIDTH; + output += 8; + } - if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback < BIT_WIDTH, SIGN_VALUES + if constexpr (remaining > 0) { + const std::vector tail_values = internal::unpack_epi32_fallback (input, remaining); - for (u32 i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi64(tail_values[i], scale); - } + for (u32 i = 0; i < remaining; i++) { + output[i] = internal::dequantize_epi64(tail_values[i], scale); } - return 0; } - - /** - * @brief Decompress multiple blocks to float using AVX2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_decompress_blocks_avx2(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const u8 *block_input = input; - f32 *block_output = output; - - for (u32 block = 0; block < blocks; block++) { - mm256_decompress_block_avx2(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; + return 0; +} + +/** +* @brief Decompress multiple blocks to float using AVX2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* +* @param input pointer to the start of the compressed data. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed float values will be stored. +* @param blocks number of blocks to decompress. +* @return int status code (0 for success). +* +* @note This function requires AVX2 support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const u8* block_input = input; + f32* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_decompress_block_avx2(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } - /** - * @brief Decompress multiple blocks to double using AVX2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_decompress_blocks_avx2(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const u8 *block_input = input; - f64 *block_output = output; - - for (u32 block = 0; block < blocks; block++) { - mm256_decompress_block_avx2(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; + return 0; +} + +/** +* @brief Decompress multiple blocks to double using AVX2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* +* @param input pointer to the start of the compressed data. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed float values will be stored. +* @param blocks number of blocks to decompress. +* @return int status code (0 for success). +* +* @note This function requires AVX2 support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const u8* block_input = input; + f64* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_decompress_block_avx2(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } + + return 0; +} } // namespace pernix #endif // PERNIX_AVX2_DECOMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h index dab8472..c51be2c 100644 --- a/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h +++ b/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h @@ -11,725 +11,726 @@ using namespace pernix::x86::internal; + namespace pernix { - namespace internal { - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - static __always_inline __m512i mm512_clamp_signed_epi32(__m512i input) { - constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); - constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm512_min_epi32(_mm512_max_epi32(input, _mm512_set1_epi32(min_value)), - _mm512_set1_epi32(max_value)); +namespace internal { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +static __always_inline __m512i mm512_clamp_signed_epi32(__m512i input) { + constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); + return _mm512_min_epi32(_mm512_max_epi32(input, _mm512_set1_epi32(min_value)), + _mm512_set1_epi32(max_value)); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +static __always_inline __m256i mm256_clamp_signed_epi32_avx512(__m256i input) { + constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); + return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), + _mm256_set1_epi32(max_value)); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +static __always_inline __m512i mm512_clamp_signed_epi64(__m512i input) { + constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); + constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); + return _mm512_min_epi64(_mm512_max_epi64(input, _mm512_set1_epi64(min_value)), + _mm512_set1_epi64(max_value)); +} + +/** +* @brief Quantize sixteen float values to 32-bit integers. +*/ + +static __always_inline __m512i mm512_quantize_ps_epi32(const __m512& input, const __m512& scale) { + const __m512 scaled = _mm512_mul_ps(input, scale); + return _mm512_cvtps_epi32(scaled); +} + +static __always_inline __m512i mm512_quantize_pd_epi64(const __m512d& input, const __m512d& scale) { + const __m512d scaled = _mm512_mul_pd(input, scale); + return _mm512_cvtpd_epi64(scaled); +} + +static __always_inline __m256i mm512_quantize_pd_epi32(const __m512d& input, const __m512d& scale) { + const __m512d scaled = _mm512_mul_pd(input, scale); + return _mm512_cvtpd_epi32(scaled); +} + +static __always_inline __m512i make_m512i_from_2x256(const __m256i a, const __m256i b) { + __m512i result = _mm512_castsi256_si512(a); + result = _mm512_inserti64x4(result, b, 1); + return result; +} + +static __always_inline __m512i make_m512i_from_4x128(const __m128i a, const __m128i b, const __m128i c, + const __m128i d) { + __m512i result = _mm512_castsi128_si512(a); + result = _mm512_inserti64x2(result, b, 1); + result = _mm512_inserti64x2(result, c, 2); + result = _mm512_inserti64x2(result, d, 3); + return result; +} + +static __always_inline __m512i make_m512i_from_8x64(const __m128i a, const __m128i b, const __m128i c, + const __m128i d, const __m128i e, + const __m128i f, const __m128i g, const __m128i h) { + const __m128i ab = _mm_unpacklo_epi64(a, b); + const __m128i cd = _mm_unpacklo_epi64(c, d); + const __m128i ef = _mm_unpacklo_epi64(e, f); + const __m128i gh = _mm_unpacklo_epi64(g, h); + + __m512i x = _mm512_castsi128_si512(ab); + x = _mm512_inserti32x4(x, cd, 1); + x = _mm512_inserti32x4(x, ef, 2); + x = _mm512_inserti32x4(x, gh, 3); + return x; +} + +static __always_inline __m256i make_m256i_from_2x128(const __m128i a, const __m128i b) { + __m256i result = _mm256_castsi128_si256(a); + result = _mm256_inserti128_si256(result, b, 1); + return result; +} + +static __always_inline __m256i make_m256i_from_4x64(const __m128i a, const __m128i b, const __m128i c, + const __m128i d) { + const __m128i ab = _mm_unpacklo_epi64(a, b); + const __m128i cd = _mm_unpacklo_epi64(c, d); + + __m256i x = _mm256_castsi128_si256(ab); + x = _mm256_inserti128_si256(x, cd, 1); + return x; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_1to8(const f32* __restrict__ input, const f32 scale, + u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr u32 iterations_64 = elements_per_block / 64; + constexpr u32 iterations_32 = (elements_per_block % 64) / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 remaining_elements = + elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + + const __m512 scale_v = _mm512_set1_ps(scale); + + if constexpr (iterations_64 > 0) { +#pragma GCC unroll 8 + for (u32 iter = 0; iter < iterations_64; ++iter) { + const __m512 source1 = _mm512_loadu_ps(input); + const __m512 source2 = _mm512_loadu_ps(input + 16); + const __m512 source3 = _mm512_loadu_ps(input + 32); + const __m512 source4 = _mm512_loadu_ps(input + 48); + + const __m512i quantized1 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source4, scale_v)); + + const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); + const __m128i converted3 = _mm512_cvtepi32_epi8(quantized3); + const __m128i converted4 = _mm512_cvtepi32_epi8(quantized4); + + const __m512i packed = + m512::mm512_pack_epi8_avx512vbmi_1to8(make_m512i_from_4x128( + converted1, converted2, converted3, converted4)); + + mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); + + input += 64; + output += 8 * BIT_WIDTH; } + } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - static __always_inline __m256i mm256_clamp_signed_epi32_avx512(__m256i input) { - constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); - constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), - _mm256_set1_epi32(max_value)); - } + if constexpr (iterations_32 > 0) { + const __m512 source1 = _mm512_loadu_ps(input); + const __m512 source2 = _mm512_loadu_ps(input + 16); - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - static __always_inline __m512i mm512_clamp_signed_epi64(__m512i input) { - constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); - constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); - return _mm512_min_epi64(_mm512_max_epi64(input, _mm512_set1_epi64(min_value)), - _mm512_set1_epi64(max_value)); - } + const __m512i quantized1 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source2, scale_v)); - /** - * @brief Quantize sixteen float values to 32-bit integers. - */ + const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); - static __always_inline __m512i mm512_quantize_ps_epi32(const __m512 &input, const __m512 &scale) { - const __m512 scaled = _mm512_mul_ps(input, scale); - return _mm512_cvtps_epi32(scaled); - } + const __m256i packed = m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_2x128( + converted1, converted2)); + mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - static __always_inline __m512i mm512_quantize_pd_epi64(const __m512d &input, const __m512d &scale) { - const __m512d scaled = _mm512_mul_pd(input, scale); - return _mm512_cvtpd_epi64(scaled); - } + input += 32; + output += 4 * BIT_WIDTH; + } - static __always_inline __m256i mm512_quantize_pd_epi32(const __m512d &input, const __m512d &scale) { - const __m512d scaled = _mm512_mul_pd(input, scale); - return _mm512_cvtpd_epi32(scaled); - } + if constexpr (iterations_16 > 0) { + const __m512 source = _mm512_loadu_ps(input); + const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); + const __m128i converted = _mm512_cvtepi32_epi8(quantized); - static __always_inline __m512i make_m512i_from_2x256(const __m256i a, const __m256i b) { - __m512i result = _mm512_castsi256_si512(a); - result = _mm512_inserti64x4(result, b, 1); - return result; - } + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(converted); + mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - static __always_inline __m512i make_m512i_from_4x128(const __m128i a, const __m128i b, const __m128i c, - const __m128i d) { - __m512i result = _mm512_castsi128_si512(a); - result = _mm512_inserti64x2(result, b, 1); - result = _mm512_inserti64x2(result, c, 2); - result = _mm512_inserti64x2(result, d, 3); - return result; - } + input += 16; + output += 2 * BIT_WIDTH; + } - static __always_inline __m512i make_m512i_from_8x64(const __m128i a, const __m128i b, const __m128i c, - const __m128i d, const __m128i e, - const __m128i f, const __m128i g, const __m128i h) { - const __m128i ab = _mm_unpacklo_epi64(a, b); - const __m128i cd = _mm_unpacklo_epi64(c, d); - const __m128i ef = _mm_unpacklo_epi64(e, f); - const __m128i gh = _mm_unpacklo_epi64(g, h); - - __m512i x = _mm512_castsi128_si512(ab); - x = _mm512_inserti32x4(x, cd, 1); - x = _mm512_inserti32x4(x, ef, 2); - x = _mm512_inserti32x4(x, gh, 3); - return x; - } + if constexpr (remaining_elements > 0) { + const __m512 source = mm512_loadu_elements_ps(remaining_elements, input); + const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); + const __m128i converted = _mm512_cvtepi32_epi8(quantized); + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(converted); - static __always_inline __m256i make_m256i_from_2x128(const __m128i a, const __m128i b) { - __m256i result = _mm256_castsi128_si256(a); - result = _mm256_inserti128_si256(result, b, 1); - return result; - } + mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } - static __always_inline __m256i make_m256i_from_4x64(const __m128i a, const __m128i b, const __m128i c, - const __m128i d) { - const __m128i ab = _mm_unpacklo_epi64(a, b); - const __m128i cd = _mm_unpacklo_epi64(c, d); + return 0; +} - __m256i x = _mm256_castsi128_si256(ab); - x = _mm256_inserti128_si256(x, cd, 1); - return x; - } +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_9to16(const f32* __restrict__ input, const f32 scale, + u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_1to8(const f32 * __restrict__ input, const f32 scale, - u8 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = + elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; - constexpr u32 iterations_64 = elements_per_block / 64; - constexpr u32 iterations_32 = (elements_per_block % 64) / 32; - constexpr u32 iterations_16 = (elements_per_block % 32) / 16; - constexpr u32 remaining_elements = - elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + const __m512 scale_v = _mm512_set1_ps(scale); + const __m256 scale_v256 = _mm256_set1_ps(scale); - const __m512 scale_v = _mm512_set1_ps(scale); + if constexpr (iterations_32 > 0) { +#pragma GCC unroll 4 + for (u32 iter = 0; iter < iterations_32; ++iter) { + const __m512 source1 = _mm512_loadu_ps(input); + const __m512 source2 = _mm512_loadu_ps(input + 16); - if constexpr (iterations_64 > 0) { -#pragma GCC unroll 8 - for (u32 iter = 0; iter < iterations_64; ++iter) { - const __m512 source1 = _mm512_loadu_ps(input); - const __m512 source2 = _mm512_loadu_ps(input + 16); - const __m512 source3 = _mm512_loadu_ps(input + 32); - const __m512 source4 = _mm512_loadu_ps(input + 48); - - const __m512i quantized1 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source4, scale_v)); - - const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); - const __m128i converted3 = _mm512_cvtepi32_epi8(quantized3); - const __m128i converted4 = _mm512_cvtepi32_epi8(quantized4); - - const __m512i packed = - m512::mm512_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (make_m512i_from_4x128( - converted1, converted2, converted3, converted4)); - - mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - - input += 64; - output += 8 * BIT_WIDTH; - } - } - - if constexpr (iterations_32 > 0) { - const __m512 source1 = _mm512_loadu_ps(input); - const __m512 source2 = _mm512_loadu_ps(input + 16); - - const __m512i quantized1 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source2, scale_v)); - - const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); - - const __m256i packed = m256::mm256_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (make_m256i_from_2x128( - converted1, converted2)); - mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - - input += 32; - output += 4 * BIT_WIDTH; - } - - if constexpr (iterations_16 > 0) { - const __m512 source = _mm512_loadu_ps(input); - const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); - const __m128i converted = _mm512_cvtepi32_epi8(quantized); - - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (converted); - mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - - input += 16; - output += 2 * BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - const __m512 source = mm512_loadu_elements_ps(remaining_elements, input); - const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); - const __m128i converted = _mm512_cvtepi32_epi8(quantized); - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (converted); - - mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; + const __m512i quantized1 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source2, scale_v)); + + const __m256i converted1 = _mm512_cvtepi32_epi16(quantized1); + const __m256i converted2 = _mm512_cvtepi32_epi16(quantized2); + + const __m512i packed = m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_2x256( + converted1, converted2)); + mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); + + input += 32; + output += 4 * BIT_WIDTH; } + } - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_9to16(const f32 * __restrict__ input, const f32 scale, - u8 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + if constexpr (iterations_16 > 0) { + const __m512 source = _mm512_loadu_ps(input); + const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); + const __m256i converted = _mm512_cvtepi32_epi16(quantized); - constexpr u32 iterations_32 = elements_per_block / 32; - constexpr u32 iterations_16 = (elements_per_block % 32) / 16; - constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = - elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(converted); + mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - const __m512 scale_v = _mm512_set1_ps(scale); - const __m256 scale_v256 = _mm256_set1_ps(scale); + input += 16; + output += 2 * BIT_WIDTH; + } - if constexpr (iterations_32 > 0) { -#pragma GCC unroll 4 - for (u32 iter = 0; iter < iterations_32; ++iter) { - const __m512 source1 = _mm512_loadu_ps(input); - const __m512 source2 = _mm512_loadu_ps(input + 16); - - const __m512i quantized1 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source2, scale_v)); - - const __m256i converted1 = _mm512_cvtepi32_epi16(quantized1); - const __m256i converted2 = _mm512_cvtepi32_epi16(quantized2); - - const __m512i packed = m512::mm512_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (make_m512i_from_2x256( - converted1, converted2)); - mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - - input += 32; - output += 4 * BIT_WIDTH; - } - } - - if constexpr (iterations_16 > 0) { - const __m512 source = _mm512_loadu_ps(input); - const __m512i quantized = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); - const __m256i converted = _mm512_cvtepi32_epi16(quantized); - - const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); - mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - - input += 16; - output += 2 * BIT_WIDTH; - } - - if constexpr (iterations_8 > 0) { - const __m256 source = _mm256_loadu_ps(input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512( - mm256_quantize_ps_epi32(source, scale_v256)); - const __m128i converted = _mm256_cvtepi32_epi16(quantized); - - const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); - mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - - input += 8; - output += BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512( - mm256_quantize_ps_epi32(source, scale_v256)); - const __m128i converted = _mm256_cvtepi32_epi16(quantized); - const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); - - mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; - } + if constexpr (iterations_8 > 0) { + const __m256 source = _mm256_loadu_ps(input); + const __m256i quantized = mm256_clamp_signed_epi32_avx512( + mm256_quantize_ps_epi32(source, scale_v256)); + const __m128i converted = _mm256_cvtepi32_epi16(quantized); - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_17to24(const f32 * __restrict__ input, const f32 scale, - u8 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); + mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - constexpr u32 iterations_16 = elements_per_block / 16; - constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + input += 8; + output += BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); + const __m256i quantized = mm256_clamp_signed_epi32_avx512( + mm256_quantize_ps_epi32(source, scale_v256)); + const __m128i converted = _mm256_cvtepi32_epi16(quantized); + const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); - const __m512 scale_v = _mm512_set1_ps(scale); - const __m256 scale_v256 = _mm256_set1_ps(scale); + mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } - if constexpr (iterations_16 > 0) { + return 0; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_17to24(const f32* __restrict__ input, const f32 scale, + u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + + const __m512 scale_v = _mm512_set1_ps(scale); + const __m256 scale_v256 = _mm256_set1_ps(scale); + + if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (u32 i = 0; i < iterations_16; ++i) { - const __m512 source = _mm512_loadu_ps(input); - const __m512i packed_input = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source, scale_v)); - - const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (packed_input); - mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; - output += 2 * BIT_WIDTH; - } - } - - if constexpr (iterations_8 > 0) { - const __m256 source = _mm256_loadu_ps(input); - const __m256i packed_input = mm256_clamp_signed_epi32_avx512( - mm256_quantize_ps_epi32(source, scale_v256)); - - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (packed_input); - mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - - input += 8; - output += BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); - const __m256i packed_input = mm256_clamp_signed_epi32_avx512( - mm256_quantize_ps_epi32(source, scale_v256)); - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (packed_input); - - mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; + for (u32 i = 0; i < iterations_16; ++i) { + const __m512 source = _mm512_loadu_ps(input); + const __m512i packed_input = mm512_clamp_signed_epi32( + mm512_quantize_ps_epi32(source, scale_v)); + + const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(packed_input); + mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); + input += 16; + output += 2 * BIT_WIDTH; } + } - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_1to8(const f64 * __restrict__ input, const f64 scale, - u8 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + if constexpr (iterations_8 > 0) { + const __m256 source = _mm256_loadu_ps(input); + const __m256i packed_input = mm256_clamp_signed_epi32_avx512( + mm256_quantize_ps_epi32(source, scale_v256)); - constexpr u32 iterations_64 = elements_per_block / 64; - constexpr u32 iterations_32 = (elements_per_block % 64) / 32; - constexpr u32 iterations_16 = (elements_per_block % 32) / 16; - constexpr u32 remaining_elements = - elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); + mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - const __m512d scale_v = _mm512_set1_pd(scale); + input += 8; + output += BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); + const __m256i packed_input = mm256_clamp_signed_epi32_avx512( + mm256_quantize_ps_epi32(source, scale_v256)); + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); + + mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } + + return 0; +} - if constexpr (iterations_64 > 0) { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ input, const f64 scale, + u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr u32 iterations_64 = elements_per_block / 64; + constexpr u32 iterations_32 = (elements_per_block % 64) / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 remaining_elements = + elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + + const __m512d scale_v = _mm512_set1_pd(scale); + + if constexpr (iterations_64 > 0) { #pragma GCC unroll 8 - for (u32 iter = 0; iter < iterations_64; ++iter) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512d source3 = _mm512_loadu_pd(input + 16); - const __m512d source4 = _mm512_loadu_pd(input + 24); - const __m512d source5 = _mm512_loadu_pd(input + 32); - const __m512d source6 = _mm512_loadu_pd(input + 40); - const __m512d source7 = _mm512_loadu_pd(input + 48); - const __m512d source8 = _mm512_loadu_pd(input + 56); - - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source4, scale_v)); - const __m512i quantized5 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source5, scale_v)); - const __m512i quantized6 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source6, scale_v)); - const __m512i quantized7 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source7, scale_v)); - const __m512i quantized8 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source8, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - const __m128i converted3 = _mm512_cvtepi64_epi8(quantized3); - const __m128i converted4 = _mm512_cvtepi64_epi8(quantized4); - const __m128i converted5 = _mm512_cvtepi64_epi8(quantized5); - const __m128i converted6 = _mm512_cvtepi64_epi8(quantized6); - const __m128i converted7 = _mm512_cvtepi64_epi8(quantized7); - const __m128i converted8 = _mm512_cvtepi64_epi8(quantized8); - - const __m512i packed = m512::mm512_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > ( - make_m512i_from_8x64(converted1, converted2, converted3, converted4, - converted5, converted6, converted7, converted8)); - - mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - - input += 64; - output += 8 * BIT_WIDTH; - } - } - - if constexpr (iterations_32 > 0) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512d source3 = _mm512_loadu_pd(input + 16); - const __m512d source4 = _mm512_loadu_pd(input + 24); - - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source4, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - const __m128i converted3 = _mm512_cvtepi64_epi8(quantized3); - const __m128i converted4 = _mm512_cvtepi64_epi8(quantized4); - - const __m256i packed = - m256::mm256_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (make_m256i_from_4x64( - converted1, converted2, converted3, converted4)); - - mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - - input += 32; - output += 4 * BIT_WIDTH; - } - - if constexpr (iterations_16 > 0) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (_mm_unpacklo_epi64( - converted1, converted2)); - - mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - - input += 16; - output += 2 * BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - constexpr u32 source1_elements = remaining_elements > 8 ? 8 : remaining_elements; - constexpr u32 source2_elements = remaining_elements > 8 ? remaining_elements - 8 : 0; - - const __m512d source1 = mm512_loadu_elements_pd(source1_elements, input); - const __m512d source2 = source2_elements > 0 - ? mm512_loadu_elements_pd(source2_elements, input + 8) - : _mm512_setzero_pd(); - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8 < BIT_WIDTH > (_mm_unpacklo_epi64( - converted1, converted2)); - - mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; + for (u32 iter = 0; iter < iterations_64; ++iter) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512d source3 = _mm512_loadu_pd(input + 16); + const __m512d source4 = _mm512_loadu_pd(input + 24); + const __m512d source5 = _mm512_loadu_pd(input + 32); + const __m512d source6 = _mm512_loadu_pd(input + 40); + const __m512d source7 = _mm512_loadu_pd(input + 48); + const __m512d source8 = _mm512_loadu_pd(input + 56); + + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source4, scale_v)); + const __m512i quantized5 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source5, scale_v)); + const __m512i quantized6 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source6, scale_v)); + const __m512i quantized7 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source7, scale_v)); + const __m512i quantized8 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source8, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); + const __m128i converted3 = _mm512_cvtepi64_epi8(quantized3); + const __m128i converted4 = _mm512_cvtepi64_epi8(quantized4); + const __m128i converted5 = _mm512_cvtepi64_epi8(quantized5); + const __m128i converted6 = _mm512_cvtepi64_epi8(quantized6); + const __m128i converted7 = _mm512_cvtepi64_epi8(quantized7); + const __m128i converted8 = _mm512_cvtepi64_epi8(quantized8); + + const __m512i packed = m512::mm512_pack_epi8_avx512vbmi_1to8( + make_m512i_from_8x64(converted1, converted2, converted3, converted4, + converted5, converted6, converted7, converted8)); + + mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); + + input += 64; + output += 8 * BIT_WIDTH; } + } - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_9to16(const f64 * __restrict__ input, const f64 scale, - u8 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + if constexpr (iterations_32 > 0) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512d source3 = _mm512_loadu_pd(input + 16); + const __m512d source4 = _mm512_loadu_pd(input + 24); + + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source4, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); + const __m128i converted3 = _mm512_cvtepi64_epi8(quantized3); + const __m128i converted4 = _mm512_cvtepi64_epi8(quantized4); + + const __m256i packed = + m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_4x64( + converted1, converted2, converted3, converted4)); + + mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); + + input += 32; + output += 4 * BIT_WIDTH; + } - constexpr u32 iterations_32 = elements_per_block / 32; - constexpr u32 iterations_16 = (elements_per_block % 32) / 16; - constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = - elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + if constexpr (iterations_16 > 0) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); - const __m512d scale_v = _mm512_set1_pd(scale); + const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - if constexpr (iterations_32 > 0) { + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(_mm_unpacklo_epi64( + converted1, converted2)); + + mm_storeu_elements_epi16(output, BIT_WIDTH, packed); + + input += 16; + output += 2 * BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + constexpr u32 source1_elements = remaining_elements > 8 ? 8 : remaining_elements; + constexpr u32 source2_elements = remaining_elements > 8 ? remaining_elements - 8 : 0; + + const __m512d source1 = mm512_loadu_elements_pd(source1_elements, input); + const __m512d source2 = source2_elements > 0 + ? mm512_loadu_elements_pd(source2_elements, input + 8) + : _mm512_setzero_pd(); + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); + + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(_mm_unpacklo_epi64( + converted1, converted2)); + + mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_9to16(const f64* __restrict__ input, const f64 scale, + u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = + elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + + const __m512d scale_v = _mm512_set1_pd(scale); + + if constexpr (iterations_32 > 0) { #pragma GCC unroll 4 - for (u32 iter = 0; iter < iterations_32; ++iter) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512d source3 = _mm512_loadu_pd(input + 16); - const __m512d source4 = _mm512_loadu_pd(input + 24); - - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source4, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); - const __m128i converted3 = _mm512_cvtepi64_epi16(quantized3); - const __m128i converted4 = _mm512_cvtepi64_epi16(quantized4); - - const __m512i packed = - m512::mm512_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (make_m512i_from_4x128( - converted1, converted2, converted3, converted4)); - - mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - - input += 32; - output += 4 * BIT_WIDTH; - } - } - - if constexpr (iterations_16 > 0) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); - - const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); - const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); - - const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (make_m256i_from_2x128( - converted1, converted2)); - - mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - - input += 16; - output += 2 * BIT_WIDTH; - } - - if constexpr (iterations_8 > 0) { - const __m512d source = _mm512_loadu_pd(input); - const __m512i quantized = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source, scale_v)); - const __m128i converted = _mm512_cvtepi64_epi16(quantized); - - const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); - mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - - input += 8; - output += BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - const __m512d source = mm512_loadu_elements_pd(remaining_elements, input); - const __m512i quantized = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source, scale_v)); - const __m128i converted = _mm512_cvtepi64_epi16(quantized); - - const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16 < BIT_WIDTH > (converted); - mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; + for (u32 iter = 0; iter < iterations_32; ++iter) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512d source3 = _mm512_loadu_pd(input + 16); + const __m512d source4 = _mm512_loadu_pd(input + 24); + + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source4, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); + const __m128i converted3 = _mm512_cvtepi64_epi16(quantized3); + const __m128i converted4 = _mm512_cvtepi64_epi16(quantized4); + + const __m512i packed = + m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_4x128( + converted1, converted2, converted3, converted4)); + + mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); + + input += 32; + output += 4 * BIT_WIDTH; } + } + + if constexpr (iterations_16 > 0) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + const __m512i quantized1 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64( + mm512_quantize_pd_epi64(source2, scale_v)); + + const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); + const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); + + const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(make_m256i_from_2x128( + converted1, converted2)); + + mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); + + input += 16; + output += 2 * BIT_WIDTH; + } - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_17to24(const f64 * __restrict__ input, const f64 scale, - u8 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + if constexpr (iterations_8 > 0) { + const __m512d source = _mm512_loadu_pd(input); + const __m512i quantized = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source, scale_v)); + const __m128i converted = _mm512_cvtepi64_epi16(quantized); - constexpr u32 iterations_16 = elements_per_block / 16; - constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); + mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - const __m512d scale_v = _mm512_set1_pd(scale); + input += 8; + output += BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const __m512d source = mm512_loadu_elements_pd(remaining_elements, input); + const __m512i quantized = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source, scale_v)); + const __m128i converted = _mm512_cvtepi64_epi16(quantized); + + const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); + mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_compress_block_avx512vbmi_17to24(const f64* __restrict__ input, const f64 scale, + u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - if constexpr (iterations_16 > 0) { + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + + const __m512d scale_v = _mm512_set1_pd(scale); + + if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (u32 i = 0; i < iterations_16; ++i) { - const __m512d source1 = _mm512_loadu_pd(input); - const __m512d source2 = _mm512_loadu_pd(input + 8); - - const __m256i quantized1 = mm256_clamp_signed_epi32_avx512( - mm512_quantize_pd_epi32(source1, scale_v)); - const __m256i quantized2 = mm256_clamp_signed_epi32_avx512( - mm512_quantize_pd_epi32(source2, scale_v)); - - const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > ( - make_m512i_from_2x256(quantized1, quantized2)); - mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - - input += 16; - output += 2 * BIT_WIDTH; - } - } - - if constexpr (iterations_8 > 0) { - const __m512d source = _mm512_loadu_pd(input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512( - mm512_quantize_pd_epi32(source, scale_v)); - - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (quantized); - mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - - input += 8; - output += BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - const __m512d source = mm512_loadu_elements_pd(remaining_elements, input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512( - mm512_quantize_pd_epi32(source, scale_v)); - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24 < BIT_WIDTH > (quantized); - - mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); - } - - return 0; - } - } // namespace internal - - /** - * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm512_compress_block_avx512vbmi(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - std::memset(output, 0, BLOCK_SIZE); - - if constexpr (BIT_WIDTH <= 8) { - return internal::mm512_compress_block_avx512vbmi_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH <= 16) { - return internal::mm512_compress_block_avx512vbmi_9to16(input, scale, output); - } else { - return internal::mm512_compress_block_avx512vbmi_17to24(input, scale, output); + for (u32 i = 0; i < iterations_16; ++i) { + const __m512d source1 = _mm512_loadu_pd(input); + const __m512d source2 = _mm512_loadu_pd(input + 8); + + const __m256i quantized1 = mm256_clamp_signed_epi32_avx512( + mm512_quantize_pd_epi32(source1, scale_v)); + const __m256i quantized2 = mm256_clamp_signed_epi32_avx512( + mm512_quantize_pd_epi32(source2, scale_v)); + + const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24( + make_m512i_from_2x256(quantized1, quantized2)); + mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); + + input += 16; + output += 2 * BIT_WIDTH; } } - /** - * @brief Compress a single block of double values using AVX-512 and AVX-512-VBMI instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code. - * - * @note This overload is declared for parity with the float path. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm512_compress_block_avx512vbmi(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - std::memset(output, 0, BLOCK_SIZE); - - if constexpr (BIT_WIDTH <= 8) { - return internal::mm512_compress_block_avx512vbmi_1to8(input, scale, output); - } else if constexpr (BIT_WIDTH <= 16) { - return internal::mm512_compress_block_avx512vbmi_9to16(input, scale, output); - } else { - return internal::mm512_compress_block_avx512vbmi_17to24(input, scale, output); - } + if constexpr (iterations_8 > 0) { + const __m512d source = _mm512_loadu_pd(input); + const __m256i quantized = mm256_clamp_signed_epi32_avx512( + mm512_quantize_pd_epi32(source, scale_v)); + + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); + mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); + + input += 8; + output += BIT_WIDTH; } - /** - * @brief Compress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm512_compress_blocks_avx512vbmi(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const f32 *block_input = input; - u8 *block_output = output; - - for (u32 block = 0; block < blocks; ++block) { - mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + if constexpr (remaining_elements > 0) { + const __m512d source = mm512_loadu_elements_pd(remaining_elements, input); + const __m256i quantized = mm256_clamp_signed_epi32_avx512( + mm512_quantize_pd_epi32(source, scale_v)); + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); - return 0; + mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); } - /** - * @brief Compress multiple blocks of double values using AVX-512 and AVX-512-VBMI instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of blocks to compress. - * @return int status code. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm512_compress_blocks_avx512vbmi(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const f64 *block_input = input; - u8 *block_output = output; - - for (u32 block = 0; block < blocks; ++block) { - mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } + return 0; +} +} // namespace internal + +/** +* @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* +* @param input pointer to the start of the input float values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX-512 and AVX-512-VBMI support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + std::memset(output, 0, BLOCK_SIZE); + + if constexpr (BIT_WIDTH <= 8) { + return internal::mm512_compress_block_avx512vbmi_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH <= 16) { + return internal::mm512_compress_block_avx512vbmi_9to16(input, scale, output); + } else { + return internal::mm512_compress_block_avx512vbmi_17to24(input, scale, output); + } +} + +/** +* @brief Compress a single block of double values using AVX-512 and AVX-512-VBMI instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @param input pointer to the start of the input double values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @return int status code. +* +* @note This overload is declared for parity with the float path. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + std::memset(output, 0, BLOCK_SIZE); + + if constexpr (BIT_WIDTH <= 8) { + return internal::mm512_compress_block_avx512vbmi_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH <= 16) { + return internal::mm512_compress_block_avx512vbmi_9to16(input, scale, output); + } else { + return internal::mm512_compress_block_avx512vbmi_17to24(input, scale, output); + } +} + +/** +* @brief Compress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* +* @param input pointer to the start of the input float values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @param blocks number of 512-bit blocks to compress. +* @return int status code (0 for success). +* +* @note This function requires AVX-512 and AVX-512-VBMI support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f32* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + mm512_compress_block_avx512vbmi(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } - return 0; + return 0; +} + +/** +* @brief Compress multiple blocks of double values using AVX-512 and AVX-512-VBMI instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @param input pointer to the start of the input double values. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where compressed bytes will be stored. +* @param blocks number of blocks to compress. +* @return int status code. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f64* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + mm512_compress_block_avx512vbmi(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; } + + return 0; +} } // namespace pernix #endif // PERNIX_AVX512VBMI_COMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h index a004be5..7969b80 100644 --- a/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h +++ b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h @@ -9,669 +9,670 @@ #include + using namespace pernix::x86::internal; namespace pernix { - namespace internal { - /** - * @brief Dequantize sixteen integer values to floats. - */ -__always_inline __m512 mm512_dequantize_epi32(const __m512i &input, const __m512 &scale) { - const __m512 converted = _mm512_cvtepi32_ps(input); - return _mm512_mul_ps(converted, scale); - } - -__always_inline __m512d mm512_dequantize_epi64(const __m512i &input, const __m512d &scale) { - const __m512d converted = _mm512_cvtepi64_pd(input); - return _mm512_mul_pd(converted, scale); - } +namespace internal { +/** +* @brief Dequantize sixteen integer values to floats. +*/ +__always_inline __m512 mm512_dequantize_epi32(const __m512i& input, const __m512& scale) { + const __m512 converted = _mm512_cvtepi32_ps(input); + return _mm512_mul_ps(converted, scale); +} + +__always_inline __m512d mm512_dequantize_epi64(const __m512i& input, const __m512d& scale) { + const __m512d converted = _mm512_cvtepi64_pd(input); + return _mm512_mul_pd(converted, scale); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict__ input, const f32 scale, + f32* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const u32 iterations_64 = elements_per_block / 64; + const u32 iterations_32 = (elements_per_block % 64) / 32; + const u32 iterations_16 = (elements_per_block % 32) / 16; + const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - + iterations_16 * 16; + + const __m512 scale_v = _mm512_set1_ps(scale); + + if constexpr (iterations_64 > 0) { +#pragma GCC unroll 8 + for (u32 i = 0; i < iterations_64; ++i) { + const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8 + (source); - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8 * __restrict__ input, const f32 scale, - f32 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + const __m512i converted1 = _mm512_cvtepi8_epi32(_mm512_castsi512_si128(unpacked)); + const __m512i converted2 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 1)); + const __m512i converted3 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 2)); + const __m512i converted4 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 3)); - const u32 iterations_64 = elements_per_block / 64; - const u32 iterations_32 = (elements_per_block % 64) / 32; - const u32 iterations_16 = (elements_per_block % 32) / 16; - const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - - iterations_16 * 16; + const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); + const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); + const __m512 dequantized3 = mm512_dequantize_epi32(converted3, scale_v); + const __m512 dequantized4 = mm512_dequantize_epi32(converted4, scale_v); - const __m512 scale_v = _mm512_set1_ps(scale); + _mm512_storeu_ps(output, dequantized1); + _mm512_storeu_ps(output + 16, dequantized2); + _mm512_storeu_ps(output + 32, dequantized3); + _mm512_storeu_ps(output + 48, dequantized4); - if constexpr (iterations_64 > 0) { -#pragma GCC unroll 8 - for (u32 i = 0; i < iterations_64; ++i) { - const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES - > - (source); - - const __m512i converted1 = _mm512_cvtepi8_epi32(_mm512_castsi512_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 1)); - const __m512i converted3 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 2)); - const __m512i converted4 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 3)); - - const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); - const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); - const __m512 dequantized3 = mm512_dequantize_epi32(converted3, scale_v); - const __m512 dequantized4 = mm512_dequantize_epi32(converted4, scale_v); - - _mm512_storeu_ps(output, dequantized1); - _mm512_storeu_ps(output + 16, dequantized2); - _mm512_storeu_ps(output + 32, dequantized3); - _mm512_storeu_ps(output + 48, dequantized4); - - output += 64; - input += 8 * BIT_WIDTH; - } - } + output += 64; + input += 8 * BIT_WIDTH; + } + } - if constexpr (iterations_32 > 0) { - const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES - > - (source); + if constexpr (iterations_32 > 0) { + const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8 + (source); - const __m512i converted1 = _mm512_cvtepi8_epi32(_mm256_castsi256_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi8_epi32(_mm256_extracti128_si256(unpacked, 1)); + const __m512i converted1 = _mm512_cvtepi8_epi32(_mm256_castsi256_si128(unpacked)); + const __m512i converted2 = _mm512_cvtepi8_epi32(_mm256_extracti128_si256(unpacked, 1)); - const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); - const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); + const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); + const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); - _mm512_storeu_ps(output, dequantized1); - _mm512_storeu_ps(output + 16, dequantized2); + _mm512_storeu_ps(output, dequantized1); + _mm512_storeu_ps(output + 16, dequantized2); - output += 32; - input += 4 * BIT_WIDTH; - } + output += 32; + input += 4 * BIT_WIDTH; + } - if constexpr (iterations_16 > 0) { - const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES - > - (source); + if constexpr (iterations_16 > 0) { + const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 + (source); - const __m512i converted = _mm512_cvtepi8_epi32(unpacked); + const __m512i converted = _mm512_cvtepi8_epi32(unpacked); - const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); + const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); - _mm512_storeu_ps(output, dequantized); + _mm512_storeu_ps(output, dequantized); - output += 16; - input += 2 * BIT_WIDTH; - } + output += 16; + input += 2 * BIT_WIDTH; + } - if constexpr (remaining_elements > 0) { - const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES - > - (source); + if constexpr (remaining_elements > 0) { + const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 + (source); - const __m512i converted = _mm512_cvtepi8_epi32(unpacked); + const __m512i converted = _mm512_cvtepi8_epi32(unpacked); - const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); + const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); - mm512_storeu_elements_ps(output, remaining_elements, dequantized); - } + mm512_storeu_elements_ps(output, remaining_elements, dequantized); + } - return 0; - } + return 0; +} - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8 * __restrict__ input, const f64 scale, - f64 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict__ input, const f64 scale, + f64* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const u32 iterations_64 = elements_per_block / 64; - const u32 iterations_32 = (elements_per_block % 64) / 32; - const u32 iterations_16 = (elements_per_block % 32) / 16; - const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - - iterations_16 * 16; + const u32 iterations_64 = elements_per_block / 64; + const u32 iterations_32 = (elements_per_block % 64) / 32; + const u32 iterations_16 = (elements_per_block % 32) / 16; + const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - + iterations_16 * 16; - const __m512d scale_v = _mm512_set1_pd(scale); + const __m512d scale_v = _mm512_set1_pd(scale); - if constexpr (iterations_64 > 0) { + if constexpr (iterations_64 > 0) { #pragma GCC unroll 8 - for (u32 i = 0; i < iterations_64; ++i) { - const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES - > - (source); - - const __m128i extracted1 = _mm512_castsi512_si128(unpacked); - const __m128i extracted2 = _mm512_extracti64x2_epi64(unpacked, 1); - const __m128i extracted3 = _mm512_extracti64x2_epi64(unpacked, 2); - const __m128i extracted4 = _mm512_extracti64x2_epi64(unpacked, 3); - - const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); - const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); - const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); - const __m512i converted5 = _mm512_cvtepi8_epi64(extracted3); - const __m512i converted6 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted3, 8)); - const __m512i converted7 = _mm512_cvtepi8_epi64(extracted4); - const __m512i converted8 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted4, 8)); - - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); - const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); - const __m512d dequantized5 = mm512_dequantize_epi64(converted5, scale_v); - const __m512d dequantized6 = mm512_dequantize_epi64(converted6, scale_v); - const __m512d dequantized7 = mm512_dequantize_epi64(converted7, scale_v); - const __m512d dequantized8 = mm512_dequantize_epi64(converted8, scale_v); - - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); - _mm512_storeu_pd(output + 16, dequantized3); - _mm512_storeu_pd(output + 24, dequantized4); - _mm512_storeu_pd(output + 32, dequantized5); - _mm512_storeu_pd(output + 40, dequantized6); - _mm512_storeu_pd(output + 48, dequantized7); - _mm512_storeu_pd(output + 56, dequantized8); - - output += 64; - input += 8 * BIT_WIDTH; - } - - if constexpr (iterations_32 > 0) { - const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES - > - (source); - - const __m128i extracted1 = _mm256_castsi256_si128(unpacked); - const __m128i extracted2 = _mm256_extracti64x2_epi64(unpacked, 1); - - const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); - const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); - const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); - - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); - const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); - - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); - _mm512_storeu_pd(output + 16, dequantized3); - _mm512_storeu_pd(output + 24, dequantized4); - - output += 32; - input += 4 * BIT_WIDTH; - } - - if constexpr (iterations_16 > 0) { - const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES - > - (source); - - const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); - - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); - - output += 16; - input += 2 * BIT_WIDTH; - } - - if constexpr (remaining_elements > 0) { - const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 < BIT_WIDTH, SIGN_VALUES - > - (source); - - const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); - - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - - mm512_storeu_elements_pd(output, remaining_elements < 8 ? remaining_elements : 8, dequantized1); - if constexpr (remaining_elements > 8) { - mm512_storeu_elements_pd(output + 8, remaining_elements - 8, dequantized2); - } - } - } + for (u32 i = 0; i < iterations_64; ++i) { + const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8 + (source); - return 0; + const __m128i extracted1 = _mm512_castsi512_si128(unpacked); + const __m128i extracted2 = _mm512_extracti64x2_epi64(unpacked, 1); + const __m128i extracted3 = _mm512_extracti64x2_epi64(unpacked, 2); + const __m128i extracted4 = _mm512_extracti64x2_epi64(unpacked, 3); + + const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); + const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); + const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); + const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); + const __m512i converted5 = _mm512_cvtepi8_epi64(extracted3); + const __m512i converted6 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted3, 8)); + const __m512i converted7 = _mm512_cvtepi8_epi64(extracted4); + const __m512i converted8 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted4, 8)); + + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); + const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); + const __m512d dequantized5 = mm512_dequantize_epi64(converted5, scale_v); + const __m512d dequantized6 = mm512_dequantize_epi64(converted6, scale_v); + const __m512d dequantized7 = mm512_dequantize_epi64(converted7, scale_v); + const __m512d dequantized8 = mm512_dequantize_epi64(converted8, scale_v); + + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); + _mm512_storeu_pd(output + 16, dequantized3); + _mm512_storeu_pd(output + 24, dequantized4); + _mm512_storeu_pd(output + 32, dequantized5); + _mm512_storeu_pd(output + 40, dequantized6); + _mm512_storeu_pd(output + 48, dequantized7); + _mm512_storeu_pd(output + 56, dequantized8); + + output += 64; + input += 8 * BIT_WIDTH; } - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8 * __restrict__ input, const f32 scale, - f32 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - constexpr u32 iterations_32 = elements_per_block / 32; - constexpr u32 iterations_16 = (elements_per_block % 32) / 16; - constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = - elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + if constexpr (iterations_32 > 0) { + const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8 + (source); - const __m512 scale_v = _mm512_set1_ps(scale); - const __m256 scale_v256 = _mm256_set1_ps(scale); + const __m128i extracted1 = _mm256_castsi256_si128(unpacked); + const __m128i extracted2 = _mm256_extracti64x2_epi64(unpacked, 1); - if constexpr (iterations_32 > 0) { -#pragma GCC unroll 4 - for (u32 i = 0; i < iterations_32; ++i) { - const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES - > - (source); + const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); + const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); + const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); + const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); - const __m512i converted1 = _mm512_cvtepi16_epi32(_mm512_castsi512_si256(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi32(_mm512_extracti32x8_epi32(unpacked, 1)); + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); + const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); - const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); - const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); + _mm512_storeu_pd(output + 16, dequantized3); + _mm512_storeu_pd(output + 24, dequantized4); - _mm512_storeu_ps(output, dequantized1); - _mm512_storeu_ps(output + 16, dequantized2); - - output += 32; - input += 4 * BIT_WIDTH; - } - } + output += 32; + input += 4 * BIT_WIDTH; + } - if constexpr (iterations_16 > 0) { - const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + if constexpr (iterations_16 > 0) { + const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 (source); - const __m512i converted = _mm512_cvtepi16_epi32(unpacked); - const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); + const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); + const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); - _mm512_storeu_ps(output, dequantized); + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - output += 16; - input += 2 * BIT_WIDTH; - } + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); + + output += 16; + input += 2 * BIT_WIDTH; + } - if constexpr (iterations_8 > 0) { - const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + if constexpr (remaining_elements > 0) { + const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 (source); - const __m256i converted = _mm256_cvtepi16_epi32(unpacked); - const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); + const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); + const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); - _mm256_storeu_ps(output, dequantized); + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - output += 8; - input += BIT_WIDTH; + mm512_storeu_elements_pd(output, remaining_elements < 8 ? remaining_elements : 8, dequantized1); + if constexpr (remaining_elements > 8) { + mm512_storeu_elements_pd(output + 8, remaining_elements - 8, dequantized2); } + } + } + + return 0; +} - if constexpr (remaining_elements > 0) { - const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict__ input, const f32 scale, + f32* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = + elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + + const __m512 scale_v = _mm512_set1_ps(scale); + const __m256 scale_v256 = _mm256_set1_ps(scale); + + if constexpr (iterations_32 > 0) { +#pragma GCC unroll 4 + for (u32 i = 0; i < iterations_32; ++i) { + const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16 (source); - const __m256i converted = _mm256_cvtepi16_epi32(unpacked); - const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); + const __m512i converted1 = _mm512_cvtepi16_epi32(_mm512_castsi512_si256(unpacked)); + const __m512i converted2 = _mm512_cvtepi16_epi32(_mm512_extracti32x8_epi32(unpacked, 1)); - mm256_storeu_elements_ps(output, remaining_elements, dequantized); - } + const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); + const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); + + _mm512_storeu_ps(output, dequantized1); + _mm512_storeu_ps(output + 16, dequantized2); - return 0; + output += 32; + input += 4 * BIT_WIDTH; } + } - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8 * __restrict__ input, const f64 scale, - f64 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + if constexpr (iterations_16 > 0) { + const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16 + (source); - constexpr u32 iterations_32 = elements_per_block / 32; - constexpr u32 iterations_16 = (elements_per_block % 32) / 16; - constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = - elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + const __m512i converted = _mm512_cvtepi16_epi32(unpacked); + const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); - const __m512d scale_v = _mm512_set1_pd(scale); + _mm512_storeu_ps(output, dequantized); - if constexpr (iterations_32 > 0) { -#pragma GCC unroll 4 - for (u32 i = 0; i < iterations_32; ++i) { - const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES - > - (source); - - const __m512i converted1 = _mm512_cvtepi16_epi64(_mm512_castsi512_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 1)); - const __m512i converted3 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 2)); - const __m512i converted4 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 3)); - - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); - const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); - - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); - _mm512_storeu_pd(output + 16, dequantized3); - _mm512_storeu_pd(output + 24, dequantized4); - - output += 32; - input += 4 * BIT_WIDTH; - } - } + output += 16; + input += 2 * BIT_WIDTH; + } - if constexpr (iterations_16 > 0) { - const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES - > - (source); + if constexpr (iterations_8 > 0) { + const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 + (source); - const __m512i converted1 = _mm512_cvtepi16_epi64(_mm256_castsi256_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi64(_mm256_extracti64x2_epi64(unpacked, 1)); + const __m256i converted = _mm256_cvtepi16_epi32(unpacked); + const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + _mm256_storeu_ps(output, dequantized); - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); + output += 8; + input += BIT_WIDTH; + } - output += 16; - input += 2 * BIT_WIDTH; - } + if constexpr (remaining_elements > 0) { + const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 + (source); - if constexpr (iterations_8 > 0) { - const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES - > - (source); + const __m256i converted = _mm256_cvtepi16_epi32(unpacked); + const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); - const __m512i converted = _mm512_cvtepi16_epi64(unpacked); + mm256_storeu_elements_ps(output, remaining_elements, dequantized); + } - const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + return 0; +} - _mm512_storeu_pd(output, dequantized); +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict__ input, const f64 scale, + f64* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - output += 8; - input += BIT_WIDTH; - } + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = + elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + + const __m512d scale_v = _mm512_set1_pd(scale); - if constexpr (remaining_elements > 0) { - const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 < BIT_WIDTH, SIGN_VALUES + if constexpr (iterations_32 > 0) { +#pragma GCC unroll 4 + for (u32 i = 0; i < iterations_32; ++i) { + const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16 (source); - const __m512i converted = _mm512_cvtepi16_epi64(unpacked); + const __m512i converted1 = _mm512_cvtepi16_epi64(_mm512_castsi512_si128(unpacked)); + const __m512i converted2 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 1)); + const __m512i converted3 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 2)); + const __m512i converted4 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 3)); - const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + const __m512d dequantized3 = mm512_dequantize_epi64(converted3, scale_v); + const __m512d dequantized4 = mm512_dequantize_epi64(converted4, scale_v); - mm512_storeu_elements_pd(output, remaining_elements, dequantized); - } + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); + _mm512_storeu_pd(output + 16, dequantized3); + _mm512_storeu_pd(output + 24, dequantized4); - return 0; + output += 32; + input += 4 * BIT_WIDTH; } + } - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8 * __restrict__ input, const f32 scale, - f32 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + if constexpr (iterations_16 > 0) { + const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16 + (source); - constexpr u32 iterations_16 = elements_per_block / 16; - constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + const __m512i converted1 = _mm512_cvtepi16_epi64(_mm256_castsi256_si128(unpacked)); + const __m512i converted2 = _mm512_cvtepi16_epi64(_mm256_extracti64x2_epi64(unpacked, 1)); - const __m512 scale_v = _mm512_set1_ps(scale); - const __m256 scale_v256 = _mm256_set1_ps(scale); + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - if constexpr (iterations_16 > 0) { -#pragma GCC unroll 2 - for (u32 i = 0; i < iterations_16; ++i) { - const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES - > - (source); + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); - const __m512 dequantized = mm512_dequantize_epi32(unpacked, scale_v); + output += 16; + input += 2 * BIT_WIDTH; + } - _mm512_storeu_ps(output, dequantized); + if constexpr (iterations_8 > 0) { + const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 + (source); - output += 16; - input += 2 * BIT_WIDTH; - } - } + const __m512i converted = _mm512_cvtepi16_epi64(unpacked); - if constexpr (iterations_8 > 0) { - const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES - > - (source); + const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); - const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); + _mm512_storeu_pd(output, dequantized); - _mm256_storeu_ps(output, dequantized); + output += 8; + input += BIT_WIDTH; + } - output += 8; - input += BIT_WIDTH; - } + if constexpr (remaining_elements > 0) { + const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 + (source); - if constexpr (remaining_elements > 0) { - const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES - > - (source); + const __m512i converted = _mm512_cvtepi16_epi64(unpacked); - const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); + const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); - mm256_storeu_elements_ps(output, remaining_elements, dequantized); - } + mm512_storeu_elements_pd(output, remaining_elements, dequantized); + } - return 0; - } + return 0; +} - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8 * __restrict__ input, const f64 scale, - f64 * __restrict__ output) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restrict__ input, const f32 scale, + f32* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 iterations_16 = elements_per_block / 16; - constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; - const __m512d scale_v = _mm512_set1_pd(scale); + const __m512 scale_v = _mm512_set1_ps(scale); + const __m256 scale_v256 = _mm256_set1_ps(scale); - if constexpr (iterations_16 > 0) { + if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (u32 i = 0; i < iterations_16; ++i) { - const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES - > - (source); + for (u32 i = 0; i < iterations_16; ++i) { + const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24 + (source); - const __m512i converted1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(unpacked)); - const __m512i converted2 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64(unpacked, 1)); + const __m512 dequantized = mm512_dequantize_epi32(unpacked, scale_v); - const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); - const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); + _mm512_storeu_ps(output, dequantized); - _mm512_storeu_pd(output, dequantized1); - _mm512_storeu_pd(output + 8, dequantized2); + output += 16; + input += 2 * BIT_WIDTH; + } + } - output += 16; - input += 2 * BIT_WIDTH; - } - } + if constexpr (iterations_8 > 0) { + const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 + (source); - if constexpr (iterations_8 > 0) { - const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES - > - (source); + const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); - const __m512i converted = _mm512_cvtepi32_epi64(unpacked); + _mm256_storeu_ps(output, dequantized); - const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + output += 8; + input += BIT_WIDTH; + } - _mm512_storeu_pd(output, dequantized); + if constexpr (remaining_elements > 0) { + const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 + (source); - output += 8; - input += BIT_WIDTH; - } + const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); + + mm256_storeu_elements_ps(output, remaining_elements, dequantized); + } + + return 0; +} - if constexpr (remaining_elements > 0) { - const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 < BIT_WIDTH, SIGN_VALUES +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restrict__ input, const f64 scale, + f64* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + + const __m512d scale_v = _mm512_set1_pd(scale); + + if constexpr (iterations_16 > 0) { +#pragma GCC unroll 2 + for (u32 i = 0; i < iterations_16; ++i) { + const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); + const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24 (source); - const __m512i converted = _mm512_cvtepi32_epi64(unpacked); + const __m512i converted1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(unpacked)); + const __m512i converted2 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64(unpacked, 1)); - const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); + const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); - mm512_storeu_elements_pd(output, remaining_elements, dequantized); - } + _mm512_storeu_pd(output, dequantized1); + _mm512_storeu_pd(output + 8, dequantized2); - return 0; - } - } // namespace internal - - /** - * @brief Decompress a single 512\-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::mm512_decompress_block_avx512vbmi_1to8( - input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::mm512_decompress_block_avx512vbmi_9to16( - input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::mm512_decompress_block_avx512vbmi_17to24( - input, scale, output); + output += 16; + input += 2 * BIT_WIDTH; } - return 0; } - /** - * @brief Decompress a single block to double values using AVX-512 and AVX-512-VBMI instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @return int status code. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::mm512_decompress_block_avx512vbmi_1to8( - input, scale, output); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::mm512_decompress_block_avx512vbmi_9to16( - input, scale, output); - } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::mm512_decompress_block_avx512vbmi_17to24( - input, scale, output); - } - return 0; + if constexpr (iterations_8 > 0) { + const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 + (source); + + const __m512i converted = _mm512_cvtepi32_epi64(unpacked); + + const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + + _mm512_storeu_pd(output, dequantized); + + output += 8; + input += BIT_WIDTH; } - /** - * @brief Decompress multiple 512\-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm512_decompress_blocks_avx512vbmi(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const u8 *block_input = input; - f32 *block_output = output; - - for (u32 block = 0; block < blocks; ++block) { - mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + if constexpr (remaining_elements > 0) { + const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 + (source); + + const __m512i converted = _mm512_cvtepi32_epi64(unpacked); - return 0; + const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); + + mm512_storeu_elements_pd(output, remaining_elements, dequantized); } - /** - * @brief Decompress multiple blocks to double values using AVX-512 and AVX-512-VBMI instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @param blocks number of blocks to decompress. - * @return int status code. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm512_decompress_blocks_avx512vbmi(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const u8 *block_input = input; - f64 *block_output = output; - - for (u32 block = 0; block < blocks; ++block) { - mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - return 0; + return 0; +} +} // namespace internal + +/** +* @brief Decompress a single 512\-bit block using AVX-512 and AVX-512-VBMI instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* +* @param input pointer to the start of the compressed block. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed float values will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX-512 and AVX-512-VBMI support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::mm512_decompress_block_avx512vbmi_1to8( + input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::mm512_decompress_block_avx512vbmi_9to16( + input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::mm512_decompress_block_avx512vbmi_17to24( + input, scale, output); + } + return 0; +} + +/** +* @brief Decompress a single block to double values using AVX-512 and AVX-512-VBMI instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @param input pointer to the start of the compressed block. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed double values will be stored. +* @return int status code. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::mm512_decompress_block_avx512vbmi_1to8( + input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::mm512_decompress_block_avx512vbmi_9to16( + input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::mm512_decompress_block_avx512vbmi_17to24( + input, scale, output); + } + return 0; +} + +/** +* @brief Decompress multiple 512\-bit blocks using AVX-512 and AVX-512-VBMI instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* +* @param input pointer to the start of the compressed data. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed float values will be stored. +* @param blocks number of 512-bit blocks to decompress. +* @return int status code (0 for success). +* +* @note This function requires AVX-512 and AVX-512-VBMI support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const u8* block_input = input; + f32* block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + mm512_decompress_block_avx512vbmi(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; +} + +/** +* @brief Decompress multiple blocks to double values using AVX-512 and AVX-512-VBMI instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @param input pointer to the start of the compressed data. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed double values will be stored. +* @param blocks number of blocks to decompress. +* @return int status code. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const u8* block_input = input; + f64* block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + mm512_decompress_block_avx512vbmi(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } + return 0; +} } // namespace pernix #endif // PERNIX_AVX512VBMI_DECOMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/compat.h b/include/pernix/x86/avx512vbmi/compat.h index df9ca4b..81f58e8 100644 --- a/include/pernix/x86/avx512vbmi/compat.h +++ b/include/pernix/x86/avx512vbmi/compat.h @@ -6,382 +6,383 @@ #include #include + namespace pernix::internal { - static __always_inline __mmask8 element_mask8(const u32 e) { - return static_cast<__mmask8>(e >= 8 ? 0xFFu : ((1u << e) - 1u)); - } +static __always_inline __mmask8 element_mask8(const u32 e) { + return static_cast<__mmask8>(e >= 8 ? 0xFFu : ((1u << e) - 1u)); +} - static __always_inline __mmask16 element_mask16(const u32 e) { - return static_cast<__mmask16>(e >= 16 ? 0xFFFFu : ((1u << e) - 1u)); - } +static __always_inline __mmask16 element_mask16(const u32 e) { + return static_cast<__mmask16>(e >= 16 ? 0xFFFFu : ((1u << e) - 1u)); +} - static __always_inline __mmask32 element_mask32(const u32 e) { - return e >= 32 ? 0xFFFFFFFFu : (1u << e) - 1u; - } +static __always_inline __mmask32 element_mask32(const u32 e) { + return e >= 32 ? 0xFFFFFFFFu : (1u << e) - 1u; +} - static __always_inline __mmask64 element_mask64(const u32 e) { - return e >= 64 ? 0xFFFFFFFFFFFFFFFFull : (1ull << e) - 1ull; - } +static __always_inline __mmask64 element_mask64(const u32 e) { + return e >= 64 ? 0xFFFFFFFFFFFFFFFFull : (1ull << e) - 1ull; +} - static __always_inline __m512i mm512_loadu_elements_epi64(const u32 e, const void *mem_addr) { +static __always_inline __m512i mm512_loadu_elements_epi64(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(i64)); - return a; + __m512i a = _mm512_setzero_si512(); + std::memcpy(&a, mem_addr, e * sizeof(i64)); + return a; #else - return _mm512_maskz_loadu_epi64(element_mask8(e), mem_addr); + return _mm512_maskz_loadu_epi64(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m256i mm256_loadu_elements_epi64(const u32 e, const void *mem_addr) { +static __always_inline __m256i mm256_loadu_elements_epi64(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(i64)); - return a; + __m256i a = _mm256_setzero_si256(); + std::memcpy(&a, mem_addr, e * sizeof(i64)); + return a; #else - return _mm256_maskz_loadu_epi64(element_mask8(e), mem_addr); + return _mm256_maskz_loadu_epi64(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m128i mm_loadu_elements_epi64(const u32 e, const void *mem_addr) { +static __always_inline __m128i mm_loadu_elements_epi64(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(i64)); - return a; + __m128i a = _mm_setzero_si128(); + std::memcpy(&a, mem_addr, e * sizeof(i64)); + return a; #else - return _mm_maskz_loadu_epi64(element_mask8(e), mem_addr); + return _mm_maskz_loadu_epi64(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m512i mm512_loadu_elements_epi32(const u32 e, const void *mem_addr) { +static __always_inline __m512i mm512_loadu_elements_epi32(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(i32)); - return a; + __m512i a = _mm512_setzero_si512(); + std::memcpy(&a, mem_addr, e * sizeof(i32)); + return a; #else - return _mm512_maskz_loadu_epi32(element_mask16(e), mem_addr); + return _mm512_maskz_loadu_epi32(element_mask16(e), mem_addr); #endif - } +} - static __always_inline __m256i mm256_loadu_elements_epi32(const u32 e, const void *mem_addr) { +static __always_inline __m256i mm256_loadu_elements_epi32(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(i32)); - return a; + __m256i a = _mm256_setzero_si256(); + std::memcpy(&a, mem_addr, e * sizeof(i32)); + return a; #else - return _mm256_maskz_loadu_epi32(element_mask8(e), mem_addr); + return _mm256_maskz_loadu_epi32(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m128i mm_loadu_elements_epi32(const u32 e, const void *mem_addr) { +static __always_inline __m128i mm_loadu_elements_epi32(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(i32)); - return a; + __m128i a = _mm_setzero_si128(); + std::memcpy(&a, mem_addr, e * sizeof(i32)); + return a; #else - return _mm_maskz_loadu_epi32(element_mask8(e), mem_addr); + return _mm_maskz_loadu_epi32(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m512i mm512_loadu_elements_epi16(const u32 e, const void *mem_addr) { +static __always_inline __m512i mm512_loadu_elements_epi16(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(i16)); - return a; + __m512i a = _mm512_setzero_si512(); + std::memcpy(&a, mem_addr, e * sizeof(i16)); + return a; #else - return _mm512_maskz_loadu_epi16(element_mask32(e), mem_addr); + return _mm512_maskz_loadu_epi16(element_mask32(e), mem_addr); #endif - } +} - static __always_inline __m256i mm256_loadu_elements_epi16(const u32 e, const void *mem_addr) { +static __always_inline __m256i mm256_loadu_elements_epi16(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(i16)); - return a; + __m256i a = _mm256_setzero_si256(); + std::memcpy(&a, mem_addr, e * sizeof(i16)); + return a; #else - return _mm256_maskz_loadu_epi16(element_mask16(e), mem_addr); + return _mm256_maskz_loadu_epi16(element_mask16(e), mem_addr); #endif - } +} - static __always_inline __m128i mm_loadu_elements_epi16(const u32 e, const void *mem_addr) { +static __always_inline __m128i mm_loadu_elements_epi16(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(i16)); - return a; + __m128i a = _mm_setzero_si128(); + std::memcpy(&a, mem_addr, e * sizeof(i16)); + return a; #else - return _mm_maskz_loadu_epi16(element_mask8(e), mem_addr); + return _mm_maskz_loadu_epi16(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m512i mm512_loadu_elements_epi8(const u32 e, const void *mem_addr) { +static __always_inline __m512i mm512_loadu_elements_epi8(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(i8)); - return a; + __m512i a = _mm512_setzero_si512(); + std::memcpy(&a, mem_addr, e * sizeof(i8)); + return a; #else - return _mm512_maskz_loadu_epi8(element_mask64(e), mem_addr); + return _mm512_maskz_loadu_epi8(element_mask64(e), mem_addr); #endif - } +} - static __always_inline __m256i mm256_loadu_elements_epi8(const u32 e, const void *mem_addr) { +static __always_inline __m256i mm256_loadu_elements_epi8(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(i8)); - return a; + __m256i a = _mm256_setzero_si256(); + std::memcpy(&a, mem_addr, e * sizeof(i8)); + return a; #else - return _mm256_maskz_loadu_epi8(element_mask32(e), mem_addr); + return _mm256_maskz_loadu_epi8(element_mask32(e), mem_addr); #endif - } +} - static __always_inline __m128i mm_loadu_elements_epi8(const u32 e, const void *mem_addr) { +static __always_inline __m128i mm_loadu_elements_epi8(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(i8)); - return a; + __m128i a = _mm_setzero_si128(); + std::memcpy(&a, mem_addr, e * sizeof(i8)); + return a; #else - return _mm_maskz_loadu_epi8(element_mask16(e), mem_addr); + return _mm_maskz_loadu_epi8(element_mask16(e), mem_addr); #endif - } +} - static __always_inline void mm512_storeu_elements_epi64(void *mem_addr, const u32 e, const __m512i a) { +static __always_inline void mm512_storeu_elements_epi64(void* mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) u8 bytes[64]; - _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(i64)); + alignas(64) u8 bytes[64]; + _mm512_storeu_si512(bytes, a); + std::memcpy(mem_addr, bytes, e * sizeof(i64)); #else - _mm512_mask_storeu_epi64(mem_addr, element_mask8(e), a); + _mm512_mask_storeu_epi64(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm256_storeu_elements_epi64(void *mem_addr, const u32 e, const __m256i a) { +static __always_inline void mm256_storeu_elements_epi64(void* mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) u8 bytes[32]; - _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(i64)); + alignas(32) u8 bytes[32]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(i64)); #else - _mm256_mask_storeu_epi64(mem_addr, element_mask8(e), a); + _mm256_mask_storeu_epi64(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm_storeu_elements_epi64(void *mem_addr, const u32 e, const __m128i a) { +static __always_inline void mm_storeu_elements_epi64(void* mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) u8 bytes[16]; - _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(i64)); + alignas(16) u8 bytes[16]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(i64)); #else - _mm_mask_storeu_epi64(mem_addr, element_mask8(e), a); + _mm_mask_storeu_epi64(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm512_storeu_elements_epi32(void *mem_addr, const u32 e, const __m512i a) { +static __always_inline void mm512_storeu_elements_epi32(void* mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) u8 bytes[64]; - _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(i32)); + alignas(64) u8 bytes[64]; + _mm512_storeu_si512(bytes, a); + std::memcpy(mem_addr, bytes, e * sizeof(i32)); #else - _mm512_mask_storeu_epi32(mem_addr, element_mask16(e), a); + _mm512_mask_storeu_epi32(mem_addr, element_mask16(e), a); #endif - } +} - static __always_inline void mm256_storeu_elements_epi32(void *mem_addr, const u32 e, const __m256i a) { +static __always_inline void mm256_storeu_elements_epi32(void* mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) u8 bytes[32]; - _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(i32)); + alignas(32) u8 bytes[32]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(i32)); #else - _mm256_mask_storeu_epi32(mem_addr, element_mask8(e), a); + _mm256_mask_storeu_epi32(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm_storeu_elements_epi32(void *mem_addr, const u32 e, const __m128i a) { +static __always_inline void mm_storeu_elements_epi32(void* mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) u8 bytes[16]; - _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(i32)); + alignas(16) u8 bytes[16]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(i32)); #else - _mm_mask_storeu_epi32(mem_addr, element_mask8(e), a); + _mm_mask_storeu_epi32(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm512_storeu_elements_epi16(void *mem_addr, const u32 e, const __m512i a) { +static __always_inline void mm512_storeu_elements_epi16(void* mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) u8 bytes[64]; - _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(i16)); + alignas(64) u8 bytes[64]; + _mm512_storeu_si512(bytes, a); + std::memcpy(mem_addr, bytes, e * sizeof(i16)); #else - _mm512_mask_storeu_epi16(mem_addr, element_mask32(e), a); + _mm512_mask_storeu_epi16(mem_addr, element_mask32(e), a); #endif - } +} - static __always_inline void mm256_storeu_elements_epi16(void *mem_addr, const u32 e, const __m256i a) { +static __always_inline void mm256_storeu_elements_epi16(void* mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) u8 bytes[32]; - _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(i16)); + alignas(32) u8 bytes[32]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(i16)); #else - _mm256_mask_storeu_epi16(mem_addr, element_mask16(e), a); + _mm256_mask_storeu_epi16(mem_addr, element_mask16(e), a); #endif - } +} - static __always_inline void mm_storeu_elements_epi16(void *mem_addr, const u32 e, const __m128i a) { +static __always_inline void mm_storeu_elements_epi16(void* mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) u8 bytes[16]; - _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(i16)); + alignas(16) u8 bytes[16]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(i16)); #else - _mm_mask_storeu_epi16(mem_addr, element_mask8(e), a); + _mm_mask_storeu_epi16(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm512_storeu_elements_epi8(void *mem_addr, const u32 e, const __m512i a) { +static __always_inline void mm512_storeu_elements_epi8(void* mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) u8 bytes[64]; - _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(i8)); + alignas(64) u8 bytes[64]; + _mm512_storeu_si512(bytes, a); + std::memcpy(mem_addr, bytes, e * sizeof(i8)); #else - _mm512_mask_storeu_epi8(mem_addr, element_mask64(e), a); + _mm512_mask_storeu_epi8(mem_addr, element_mask64(e), a); #endif - } +} - static __always_inline void mm256_storeu_elements_epi8(void *mem_addr, const u32 e, const __m256i a) { +static __always_inline void mm256_storeu_elements_epi8(void* mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) u8 bytes[32]; - _mm256_storeu_si256(reinterpret_cast<__m256i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(i8)); + alignas(32) u8 bytes[32]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(i8)); #else - _mm256_mask_storeu_epi8(mem_addr, element_mask32(e), a); + _mm256_mask_storeu_epi8(mem_addr, element_mask32(e), a); #endif - } +} - static __always_inline void mm_storeu_elements_epi8(void *mem_addr, const u32 e, const __m128i a) { +static __always_inline void mm_storeu_elements_epi8(void* mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) u8 bytes[16]; - _mm_storeu_si128(reinterpret_cast<__m128i *>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(i8)); + alignas(16) u8 bytes[16]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); + std::memcpy(mem_addr, bytes, e * sizeof(i8)); #else - _mm_mask_storeu_epi8(mem_addr, element_mask16(e), a); + _mm_mask_storeu_epi8(mem_addr, element_mask16(e), a); #endif - } +} - static __always_inline __m512 mm512_loadu_elements_ps(const u32 e, const void *mem_addr) { +static __always_inline __m512 mm512_loadu_elements_ps(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512 a = _mm512_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(f32)); - return a; + __m512 a = _mm512_setzero_ps(); + std::memcpy(&a, mem_addr, e * sizeof(f32)); + return a; #else - return _mm512_maskz_loadu_ps(element_mask16(e), mem_addr); + return _mm512_maskz_loadu_ps(element_mask16(e), mem_addr); #endif - } +} - static __always_inline __m256 mm256_loadu_elements_ps(const u32 e, const void *mem_addr) { +static __always_inline __m256 mm256_loadu_elements_ps(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256 a = _mm256_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(f32)); - return a; + __m256 a = _mm256_setzero_ps(); + std::memcpy(&a, mem_addr, e * sizeof(f32)); + return a; #else - return _mm256_maskz_loadu_ps(element_mask8(e), mem_addr); + return _mm256_maskz_loadu_ps(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m128 mm_loadu_elements_ps(const u32 e, const void *mem_addr) { +static __always_inline __m128 mm_loadu_elements_ps(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128 a = _mm_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(f32)); - return a; + __m128 a = _mm_setzero_ps(); + std::memcpy(&a, mem_addr, e * sizeof(f32)); + return a; #else - return _mm_maskz_loadu_ps(element_mask8(e), mem_addr); + return _mm_maskz_loadu_ps(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m512d mm512_loadu_elements_pd(const u32 e, const void *mem_addr) { +static __always_inline __m512d mm512_loadu_elements_pd(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m512d a = _mm512_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(f64)); - return a; + __m512d a = _mm512_setzero_pd(); + std::memcpy(&a, mem_addr, e * sizeof(f64)); + return a; #else - return _mm512_maskz_loadu_pd(element_mask8(e), mem_addr); + return _mm512_maskz_loadu_pd(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m256d mm256_loadu_elements_pd(const u32 e, const void *mem_addr) { +static __always_inline __m256d mm256_loadu_elements_pd(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m256d a = _mm256_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(f64)); - return a; + __m256d a = _mm256_setzero_pd(); + std::memcpy(&a, mem_addr, e * sizeof(f64)); + return a; #else - return _mm256_maskz_loadu_pd(element_mask8(e), mem_addr); + return _mm256_maskz_loadu_pd(element_mask8(e), mem_addr); #endif - } +} - static __always_inline __m128d mm_loadu_elements_pd(const u32 e, const void *mem_addr) { +static __always_inline __m128d mm_loadu_elements_pd(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - __m128d a = _mm_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(f64)); - return a; + __m128d a = _mm_setzero_pd(); + std::memcpy(&a, mem_addr, e * sizeof(f64)); + return a; #else - return _mm_maskz_loadu_pd(element_mask8(e), mem_addr); + return _mm_maskz_loadu_pd(element_mask8(e), mem_addr); #endif - } +} - static __always_inline void mm512_storeu_elements_ps(void *mem_addr, const u32 e, const __m512 a) { +static __always_inline void mm512_storeu_elements_ps(void* mem_addr, const u32 e, const __m512 a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) f32 values[16]; - _mm512_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(f32)); + alignas(64) f32 values[16]; + _mm512_storeu_ps(values, a); + std::memcpy(mem_addr, values, e * sizeof(f32)); #else - _mm512_mask_storeu_ps(mem_addr, element_mask16(e), a); + _mm512_mask_storeu_ps(mem_addr, element_mask16(e), a); #endif - } +} - static __always_inline void mm256_storeu_elements_ps(void *mem_addr, const u32 e, const __m256 a) { +static __always_inline void mm256_storeu_elements_ps(void* mem_addr, const u32 e, const __m256 a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) f32 values[8]; - _mm256_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(f32)); + alignas(32) f32 values[8]; + _mm256_storeu_ps(values, a); + std::memcpy(mem_addr, values, e * sizeof(f32)); #else - _mm256_mask_storeu_ps(mem_addr, element_mask8(e), a); + _mm256_mask_storeu_ps(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm_storeu_elements_ps(void *mem_addr, const u32 e, const __m128 a) { +static __always_inline void mm_storeu_elements_ps(void* mem_addr, const u32 e, const __m128 a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) f32 values[4]; - _mm_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(f32)); + alignas(16) f32 values[4]; + _mm_storeu_ps(values, a); + std::memcpy(mem_addr, values, e * sizeof(f32)); #else - _mm_mask_storeu_ps(mem_addr, element_mask8(e), a); + _mm_mask_storeu_ps(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm512_storeu_elements_pd(void *mem_addr, const u32 e, const __m512d a) { +static __always_inline void mm512_storeu_elements_pd(void* mem_addr, const u32 e, const __m512d a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) f64 values[8]; - _mm512_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(f64)); + alignas(64) f64 values[8]; + _mm512_storeu_pd(values, a); + std::memcpy(mem_addr, values, e * sizeof(f64)); #else - _mm512_mask_storeu_pd(mem_addr, element_mask8(e), a); + _mm512_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm256_storeu_elements_pd(void *mem_addr, const u32 e, const __m256d a) { +static __always_inline void mm256_storeu_elements_pd(void* mem_addr, const u32 e, const __m256d a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) f64 values[4]; - _mm256_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(f64)); + alignas(32) f64 values[4]; + _mm256_storeu_pd(values, a); + std::memcpy(mem_addr, values, e * sizeof(f64)); #else - _mm256_mask_storeu_pd(mem_addr, element_mask8(e), a); + _mm256_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif - } +} - static __always_inline void mm_storeu_elements_pd(void *mem_addr, const u32 e, const __m128d a) { +static __always_inline void mm_storeu_elements_pd(void* mem_addr, const u32 e, const __m128d a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) f64 values[2]; - _mm_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(f64)); + alignas(16) f64 values[2]; + _mm_storeu_pd(values, a); + std::memcpy(mem_addr, values, e * sizeof(f64)); #else - _mm_mask_storeu_pd(mem_addr, element_mask8(e), a); + _mm_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif - } +} } #endif //PERNIX_AVX512_COMPAT_H diff --git a/include/pernix/x86/avx512vbmi/packing.h b/include/pernix/x86/avx512vbmi/packing.h index d4052eb..301afb6 100644 --- a/include/pernix/x86/avx512vbmi/packing.h +++ b/include/pernix/x86/avx512vbmi/packing.h @@ -4,328 +4,329 @@ #include #include -namespace pernix::internal { - namespace m128 { - /** - * @brief Pack 8 16-bit values for bit widths 9 through 16 using VBMI. - */ - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i &input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = pack_tables_avx512_16; - const __m128i maskv = _mm_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i permuted1 = _mm_permutexvar_epi16(tables::get_permute1(), masked); - const __m128i permuted2 = _mm_permutexvar_epi16(tables::get_permute2(), masked); - - const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm_or_si128(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m128i permuted1 = _mm_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m128i permuted2 = _mm_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m128i permuted3 = _mm_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); - - const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); - } - } - } - - /** - * @brief Pack 16 8-bit values for bit widths 1 through 8 using VBMI. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i &input) { - if constexpr (BIT_WIDTH == 8) { - return input; - } else { - const __m128i maskv = _mm_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm_set1_epi16(static_cast(_mm_cmpgt_epi8_mask(masked, _mm_setzero_si128()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m128i shifted = _mm_srli_epi16(masked, 6); - const __m128i combined = _mm_or_si128(masked, shifted); - - const __m128i shifted2 = _mm_srli_epi32(combined, 12); - const __m128i combined2 = _mm_or_si128(shifted2, combined); - - return _mm_cvtepi32_epi8(combined2); - } else if constexpr (BIT_WIDTH == 3) { - const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); - - const __m128i pair6 = _mm_or_si128(even, _mm_srli_epi16(odd, 5)); - const __m128i packed12 = _mm_or_si128(pair6, _mm_srli_epi32(pair6, 10)); - - return m128::mm_pack_epi16_avx512vbmi_9to16<12>(_mm_cvtepi32_epi16(packed12)); - } else if constexpr (BIT_WIDTH == 4) { - const __m128i shifted = _mm_srli_epi16(masked, 4); - const __m128i combined = _mm_or_si128(masked, shifted); - - return _mm_cvtepi16_epi8(combined); - } else { - const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); - - const __m128i shifted = _mm_or_si128(even, _mm_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } - } - } - - /** - * @brief Pack 4 32-bit values for bit widths 17 through 24 using VBMI. - */ - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i &input) { - using tables = pack_tables_avx512_24; - - const __m128i maskv = _mm_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - const __m128 permuted1 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute1()); - const __m128 permuted2 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute2()); - const __m128 permuted3 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute3()); - - const __m128i shifted1 = _mm_sllv_epi32(_mm_castps_si128(permuted1), tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi32(_mm_castps_si128(permuted2), tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi32(_mm_castps_si128(permuted3), tables::get_shift3()); +namespace pernix::internal { +namespace m128 { +/** +* @brief Pack 8 16-bit values for bit widths 9 through 16 using VBMI. +*/ +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = pack_tables_avx512_16; + const __m128i maskv = _mm_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); + const __m128i masked = _mm_and_si128(input, maskv); + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m128i permuted1 = _mm_permutexvar_epi16(tables::get_permute1(), masked); + const __m128i permuted2 = _mm_permutexvar_epi16(tables::get_permute2(), masked); + + const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); + const __m128i shifted2 = _mm_srlv_epi16(permuted2, tables::get_shift2()); + + return _mm_or_si128(shifted1, shifted2); + } else { + const auto [mask1, mask2, mask3] = tables::get_permute_masks(); + + const __m128i permuted1 = _mm_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); + const __m128i permuted2 = _mm_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); + const __m128i permuted3 = _mm_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); + + const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); + const __m128i shifted3 = _mm_srlv_epi16(permuted3, tables::get_shift3()); return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); } - } // namespace m128 - - namespace m256 { - /** - * @brief Pack 16 16-bit values for bit widths 9 through 16 using VBMI. - */ - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i &input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = pack_tables_avx512_16; - const __m256i maskv = _mm256_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m256i permuted1 = _mm256_permutexvar_epi16(tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_permutexvar_epi16(tables::get_permute2(), masked); - - const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm256_or_si256(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); - - const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); - } - } + } +} + +/** +* @brief Pack 16 8-bit values for bit widths 1 through 8 using VBMI. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + const __m128i maskv = _mm_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); + const __m128i masked = _mm_and_si128(input, maskv); + + if constexpr (BIT_WIDTH == 1) { + return _mm_set1_epi16(static_cast(_mm_cmpgt_epi8_mask(masked, _mm_setzero_si128()))); + } else if constexpr (BIT_WIDTH == 2) { + const __m128i shifted = _mm_srli_epi16(masked, 6); + const __m128i combined = _mm_or_si128(masked, shifted); + + const __m128i shifted2 = _mm_srli_epi32(combined, 12); + const __m128i combined2 = _mm_or_si128(shifted2, combined); + + return _mm_cvtepi32_epi8(combined2); + } else if constexpr (BIT_WIDTH == 3) { + const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); + const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); + + const __m128i pair6 = _mm_or_si128(even, _mm_srli_epi16(odd, 5)); + const __m128i packed12 = _mm_or_si128(pair6, _mm_srli_epi32(pair6, 10)); + + return m128::mm_pack_epi16_avx512vbmi_9to16<12>(_mm_cvtepi32_epi16(packed12)); + } else if constexpr (BIT_WIDTH == 4) { + const __m128i shifted = _mm_srli_epi16(masked, 4); + const __m128i combined = _mm_or_si128(masked, shifted); + + return _mm_cvtepi16_epi8(combined); + } else { + const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); + const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); + + const __m128i shifted = _mm_or_si128(even, _mm_srli_epi16(odd, 8 - BIT_WIDTH)); + return mm_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); } - - /** - * @brief Pack 32 8-bit values for bit widths 1 through 8 using VBMI. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i &input) { - if constexpr (BIT_WIDTH == 8) { - return input; - } else { - const __m256i maskv = _mm256_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm256_set1_epi32( - static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m256i shifted = _mm256_srli_epi16(masked, 6); - const __m256i combined = _mm256_or_si256(masked, shifted); - - const __m256i shifted2 = _mm256_srli_epi32(combined, 12); - const __m256i combined2 = _mm256_or_si256(shifted2, combined); - - return _mm256_castsi128_si256(_mm256_cvtepi32_epi8(combined2)); - } else if constexpr (BIT_WIDTH == 3) { - const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); - - const __m256i pair6 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 5)); - const __m256i packed12 = _mm256_or_si256(pair6, _mm256_srli_epi32(pair6, 10)); - - return m256::mm256_pack_epi16_avx512vbmi_9to16<12>( - _mm256_castsi128_si256(_mm256_cvtepi32_epi16(packed12))); - } else if constexpr (BIT_WIDTH == 4) { - const __m256i shifted = _mm256_srli_epi16(masked, 4); - const __m256i combined = _mm256_or_si256(masked, shifted); - - return _mm256_castsi128_si256(_mm256_cvtepi16_epi8(combined)); - } else { - const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); - - const __m256i shifted = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm256_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } - } - } - - /** - * @brief Pack 8 32-bit values for bit widths 17 through 24 using VBMI. - */ - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i &input) { - using tables = pack_tables_avx512_24; - - const __m256i maskv = _mm256_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - const __m256i permuted1 = _mm256_permutexvar_epi32(tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_permutexvar_epi32(tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_permutexvar_epi32(tables::get_permute3(), masked); - - const __m256i shifted1 = _mm256_sllv_epi32(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi32(permuted2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi32(permuted3, tables::get_shift3()); + } +} + +/** +* @brief Pack 4 32-bit values for bit widths 17 through 24 using VBMI. +*/ +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i& input) { + using tables = pack_tables_avx512_24; + + const __m128i maskv = _mm_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); + const __m128i masked = _mm_and_si128(input, maskv); + + const __m128 permuted1 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute1()); + const __m128 permuted2 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute2()); + const __m128 permuted3 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute3()); + + const __m128i shifted1 = _mm_sllv_epi32(_mm_castps_si128(permuted1), tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi32(_mm_castps_si128(permuted2), tables::get_shift2()); + const __m128i shifted3 = _mm_srlv_epi32(_mm_castps_si128(permuted3), tables::get_shift3()); + + return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); +} +} // namespace m128 + +namespace m256 { +/** +* @brief Pack 16 16-bit values for bit widths 9 through 16 using VBMI. +*/ +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = pack_tables_avx512_16; + const __m256i maskv = _mm256_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); + const __m256i masked = _mm256_and_si256(input, maskv); + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m256i permuted1 = _mm256_permutexvar_epi16(tables::get_permute1(), masked); + const __m256i permuted2 = _mm256_permutexvar_epi16(tables::get_permute2(), masked); + + const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); + const __m256i shifted2 = _mm256_srlv_epi16(permuted2, tables::get_shift2()); + + return _mm256_or_si256(shifted1, shifted2); + } else { + const auto [mask1, mask2, mask3] = tables::get_permute_masks(); + + const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); + const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); + const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); + + const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); + const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); + const __m256i shifted3 = _mm256_srlv_epi16(permuted3, tables::get_shift3()); return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); } - } // namespace m256 - - namespace m512 { - /** - * @brief Pack 32 16-bit values for bit widths 9 through 16 using VBMI. - */ - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i &input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = pack_tables_avx512_16; - const __m512i maskv = _mm512_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m512i permuted1 = _mm512_permutexvar_epi16(tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_permutexvar_epi16(tables::get_permute2(), masked); - - const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm512_or_si512(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); - - const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); - const __m512i shifted3 = _mm512_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); - } - } - } - - /** - * @brief Pack 64 8-bit values for bit widths 1 through 8 using VBMI. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i &input) { - if constexpr (BIT_WIDTH == 8) { - return input; - } else { - const __m512i maskv = _mm512_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm512_set1_epi64( - static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m512i shifted = _mm512_srli_epi16(masked, 6); - const __m512i combined = _mm512_or_si512(masked, shifted); - - const __m512i shifted2 = _mm512_srli_epi32(combined, 12); - const __m512i combined2 = _mm512_or_si512(shifted2, combined); - - return _mm512_castsi128_si512(_mm512_cvtepi32_epi8(combined2)); - } else if constexpr (BIT_WIDTH == 3) { - const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); - - const __m512i pair6 = _mm512_or_si512(even, _mm512_srli_epi16(odd, 5)); - const __m512i packed12 = _mm512_or_si512(pair6, _mm512_srli_epi32(pair6, 10)); - - return _mm512_castsi256_si512( - m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm512_cvtepi32_epi16(packed12))); - } else if constexpr (BIT_WIDTH == 4) { - const __m512i shifted = _mm512_srli_epi16(masked, 4); - const __m512i combined = _mm512_or_si512(masked, shifted); - - return _mm512_castsi256_si512(_mm512_cvtepi16_epi8(combined)); - } else { - const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); - - const __m512i shifted = _mm512_or_si512(even, _mm512_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm512_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } - } + } +} + +/** +* @brief Pack 32 8-bit values for bit widths 1 through 8 using VBMI. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + const __m256i maskv = _mm256_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); + const __m256i masked = _mm256_and_si256(input, maskv); + + if constexpr (BIT_WIDTH == 1) { + return _mm256_set1_epi32( + static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); + } else if constexpr (BIT_WIDTH == 2) { + const __m256i shifted = _mm256_srli_epi16(masked, 6); + const __m256i combined = _mm256_or_si256(masked, shifted); + + const __m256i shifted2 = _mm256_srli_epi32(combined, 12); + const __m256i combined2 = _mm256_or_si256(shifted2, combined); + + return _mm256_castsi128_si256(_mm256_cvtepi32_epi8(combined2)); + } else if constexpr (BIT_WIDTH == 3) { + const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); + const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); + + const __m256i pair6 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 5)); + const __m256i packed12 = _mm256_or_si256(pair6, _mm256_srli_epi32(pair6, 10)); + + return m256::mm256_pack_epi16_avx512vbmi_9to16<12>( + _mm256_castsi128_si256(_mm256_cvtepi32_epi16(packed12))); + } else if constexpr (BIT_WIDTH == 4) { + const __m256i shifted = _mm256_srli_epi16(masked, 4); + const __m256i combined = _mm256_or_si256(masked, shifted); + + return _mm256_castsi128_si256(_mm256_cvtepi16_epi8(combined)); + } else { + const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); + const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); + + const __m256i shifted = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); + return mm256_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); } - - /** - * @brief Pack 16 32-bit values for bit widths 17 through 24 using VBMI. - */ - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i &input) { - using tables = pack_tables_avx512_24; - - const __m512i maskv = _mm512_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - const __m512i permuted1 = _mm512_permutexvar_epi32(tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_permutexvar_epi32(tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_permutexvar_epi32(tables::get_permute3(), masked); - - const __m512i shifted1 = _mm512_sllv_epi32(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi32(permuted2, tables::get_shift2()); - const __m512i shifted3 = _mm512_srlv_epi32(permuted3, tables::get_shift3()); + } +} + +/** +* @brief Pack 8 32-bit values for bit widths 17 through 24 using VBMI. +*/ +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i& input) { + using tables = pack_tables_avx512_24; + + const __m256i maskv = _mm256_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); + const __m256i masked = _mm256_and_si256(input, maskv); + + const __m256i permuted1 = _mm256_permutexvar_epi32(tables::get_permute1(), masked); + const __m256i permuted2 = _mm256_permutexvar_epi32(tables::get_permute2(), masked); + const __m256i permuted3 = _mm256_permutexvar_epi32(tables::get_permute3(), masked); + + const __m256i shifted1 = _mm256_sllv_epi32(permuted1, tables::get_shift1()); + const __m256i shifted2 = _mm256_sllv_epi32(permuted2, tables::get_shift2()); + const __m256i shifted3 = _mm256_srlv_epi32(permuted3, tables::get_shift3()); + + return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); +} +} // namespace m256 + +namespace m512 { +/** +* @brief Pack 32 16-bit values for bit widths 9 through 16 using VBMI. +*/ +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = pack_tables_avx512_16; + const __m512i maskv = _mm512_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); + const __m512i masked = _mm512_and_si512(input, maskv); + + if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m512i permuted1 = _mm512_permutexvar_epi16(tables::get_permute1(), masked); + const __m512i permuted2 = _mm512_permutexvar_epi16(tables::get_permute2(), masked); + + const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); + const __m512i shifted2 = _mm512_srlv_epi16(permuted2, tables::get_shift2()); + + return _mm512_or_si512(shifted1, shifted2); + } else { + const auto [mask1, mask2, mask3] = tables::get_permute_masks(); + + const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); + const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); + const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); + + const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); + const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); + const __m512i shifted3 = _mm512_srlv_epi16(permuted3, tables::get_shift3()); return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); } - } // namespace m512 + } +} + +/** +* @brief Pack 64 8-bit values for bit widths 1 through 8 using VBMI. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + const __m512i maskv = _mm512_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); + const __m512i masked = _mm512_and_si512(input, maskv); + + if constexpr (BIT_WIDTH == 1) { + return _mm512_set1_epi64( + static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); + } else if constexpr (BIT_WIDTH == 2) { + const __m512i shifted = _mm512_srli_epi16(masked, 6); + const __m512i combined = _mm512_or_si512(masked, shifted); + + const __m512i shifted2 = _mm512_srli_epi32(combined, 12); + const __m512i combined2 = _mm512_or_si512(shifted2, combined); + + return _mm512_castsi128_si512(_mm512_cvtepi32_epi8(combined2)); + } else if constexpr (BIT_WIDTH == 3) { + const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); + const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); + + const __m512i pair6 = _mm512_or_si512(even, _mm512_srli_epi16(odd, 5)); + const __m512i packed12 = _mm512_or_si512(pair6, _mm512_srli_epi32(pair6, 10)); + + return _mm512_castsi256_si512( + m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm512_cvtepi32_epi16(packed12))); + } else if constexpr (BIT_WIDTH == 4) { + const __m512i shifted = _mm512_srli_epi16(masked, 4); + const __m512i combined = _mm512_or_si512(masked, shifted); + + return _mm512_castsi256_si512(_mm512_cvtepi16_epi8(combined)); + } else { + const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); + const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); + + const __m512i shifted = _mm512_or_si512(even, _mm512_srli_epi16(odd, 8 - BIT_WIDTH)); + return mm512_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); + } + } +} + +/** +* @brief Pack 16 32-bit values for bit widths 17 through 24 using VBMI. +*/ +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i& input) { + using tables = pack_tables_avx512_24; + + const __m512i maskv = _mm512_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); + const __m512i masked = _mm512_and_si512(input, maskv); + + const __m512i permuted1 = _mm512_permutexvar_epi32(tables::get_permute1(), masked); + const __m512i permuted2 = _mm512_permutexvar_epi32(tables::get_permute2(), masked); + const __m512i permuted3 = _mm512_permutexvar_epi32(tables::get_permute3(), masked); + + const __m512i shifted1 = _mm512_sllv_epi32(permuted1, tables::get_shift1()); + const __m512i shifted2 = _mm512_sllv_epi32(permuted2, tables::get_shift2()); + const __m512i shifted3 = _mm512_srlv_epi32(permuted3, tables::get_shift3()); + + return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); +} +} // namespace m512 } // namespace pernix::internal #endif // PERNIX_AVX512VBMI_PACKING_H diff --git a/include/pernix/x86/avx512vbmi/unpacking.h b/include/pernix/x86/avx512vbmi/unpacking.h index e66f9ec..2f6579a 100644 --- a/include/pernix/x86/avx512vbmi/unpacking.h +++ b/include/pernix/x86/avx512vbmi/unpacking.h @@ -5,496 +5,497 @@ #include namespace pernix::internal { - namespace m128 { - constexpr __mmask16 kAlternateByteMask16 = 0xAAAAULL; +namespace m128 { +constexpr __mmask16 kAlternateByteMask16 = 0xAAAAULL; + __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0x00ff); - const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); - const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); - return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); - } + const __m128i mask = _mm_set1_epi16(0x00ff); + const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); + const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); + return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); +} __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0xff00); - const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); - const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); - return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); - } + const __m128i mask = _mm_set1_epi16(0xff00); + const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); + const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); + return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); +} __always_inline static __m128i _mm_slli_epi8(const __m128i a, const i8 imm8) { - return _mm_sllv_epi8(a, _mm_set1_epi8(imm8)); - } + return _mm_sllv_epi8(a, _mm_set1_epi8(imm8)); +} __always_inline static __m128i _mm_srli_epi8(const __m128i a, const int imm8) { - const __m128i lo_mask = _mm_set1_epi16(0x00ff); - const __m128i hi_mask = _mm_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); + const __m128i lo_mask = _mm_set1_epi16(0x00ff); + const __m128i hi_mask = _mm_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); - const __m128i lo = _mm_srl_epi16(_mm_and_si128(a, lo_mask), shift); - const __m128i hi = _mm_and_si128(_mm_srl_epi16(a, shift), hi_mask); + const __m128i lo = _mm_srl_epi16(_mm_and_si128(a, lo_mask), shift); + const __m128i hi = _mm_and_si128(_mm_srl_epi16(a, shift), hi_mask); - return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); - } + return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); +} __always_inline static __m128i _mm_srai_epi8(const __m128i a, const i8 imm8) { - const __m128i lo_mask = _mm_set1_epi16(0x00ff); - const __m128i hi_mask = _mm_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); - - const __m128i hi = _mm_and_si128(_mm_sra_epi16(a, shift), hi_mask); - - const __m128i lo_as_hi = _mm_slli_epi16(_mm_and_si128(a, lo_mask), 8); - const __m128i lo = _mm_and_si128(_mm_srli_epi16(_mm_sra_epi16(lo_as_hi, shift), 8), lo_mask); - - return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); - } - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i &input) { - if constexpr (BIT_WIDTH == 8) { - return input; + const __m128i lo_mask = _mm_set1_epi16(0x00ff); + const __m128i hi_mask = _mm_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); + + const __m128i hi = _mm_and_si128(_mm_sra_epi16(a, shift), hi_mask); + + const __m128i lo_as_hi = _mm_slli_epi16(_mm_and_si128(a, lo_mask), 8); + const __m128i lo = _mm_and_si128(_mm_srli_epi16(_mm_sra_epi16(lo_as_hi, shift), 8), lo_mask); + + return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i& input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + if constexpr (BIT_WIDTH == 1) { + const auto value = static_cast<__mmask16>(_mm_cvtsi128_si64(input)); + const __m128i source = _mm_movm_epi8(value); + const __m128i unpacked = _mm_abs_epi8(source); + return unpacked; + } else if constexpr (BIT_WIDTH == 2) { + __m128i values_shift0 = input; + __m128i values_shift2 = _mm_srli_epi16(values_shift0, 2); + const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); + const __m128i values_shift6 = _mm_srli_epi16(values_shift0, 6); + + __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift2); + values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift2); + values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); + + interleave_tmp = _mm_unpacklo_epi8(values_shift4, values_shift6); + values_shift2 = _mm_unpackhi_epi8(values_shift4, values_shift6); + values_shift2 = _mm_unpacklo_epi64(interleave_tmp, values_shift2); + + interleave_tmp = _mm_unpacklo_epi16(values_shift0, values_shift2); + values_shift0 = _mm_unpackhi_epi16(values_shift0, values_shift2); + values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); + values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); + + values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0303)); + + return values_shift0; + } else if constexpr (BIT_WIDTH == 4) { + __m128i values_shift0 = input; + const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); + + const __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift4); + values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift4); + values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); + values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); + + values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0F0F)); + + return values_shift0; + } else { + using tables = unpack_tables_avx512_8; + + const __m128i permuted1 = _mm_permutexvar_epi8(tables::get_permute1(), input); + const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); + + const __m128i shifted1 = _mm_srlv_epi8(permuted1, tables::get_shift1()); + const __m128i shifted2 = _mm_sllv_epi8(permuted2, tables::get_shift2()); + + const __mmask16 spill_mask = _mm_cmpneq_epi8_mask(tables::get_shift2(), _mm_setzero_si128()); + __m128i combined = _mm_or_si128(shifted1, _mm_maskz_mov_epi8(spill_mask, shifted2)); + + constexpr u32 shift = 8 - BIT_WIDTH; + combined = _mm_slli_epi8(combined, shift); + if (SIGN_VALUES) { + combined = _mm_srai_epi8(combined, shift); } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask16>(_mm_cvtsi128_si64(input)); - const __m128i source = _mm_movm_epi8(value); - const __m128i unpacked = _mm_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m128i values_shift0 = input; - __m128i values_shift2 = _mm_srli_epi16(values_shift0, 2); - const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); - const __m128i values_shift6 = _mm_srli_epi16(values_shift0, 6); - - __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - - interleave_tmp = _mm_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm_unpacklo_epi64(interleave_tmp, values_shift2); - - interleave_tmp = _mm_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); - - values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0303)); - - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m128i values_shift0 = input; - const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); - - const __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); - - values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0F0F)); - - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m128i permuted1 = _mm_permutexvar_epi8(tables::get_permute1(), input); - const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); - - const __m128i shifted1 = _mm_srlv_epi8(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask16 spill_mask = _mm_cmpneq_epi8_mask(tables::get_shift2(), _mm_setzero_si128()); - __m128i combined = _mm_or_si128(shifted1, _mm_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr u32 shift = 8 - BIT_WIDTH; - combined = _mm_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm_srai_epi8(combined, shift); - } else { - combined = _mm_srli_epi8(combined, shift); - } - - return combined; - } + combined = _mm_srli_epi8(combined, shift); } - } - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i &input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = unpack_tables_avx512_16; + return combined; + } + } +} - const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute1(), input); +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i& input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = unpack_tables_avx512_16; - __m128i shifted = _mm_srlv_epi16(permuted, tables::get_shift1()); + const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute1(), input); - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); - const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); - shifted = _mm_or_si128(shifted, shifted2); - } + __m128i shifted = _mm_srlv_epi16(permuted, tables::get_shift1()); - constexpr u32 shift = 16 - BIT_WIDTH; - shifted = _mm_slli_epi16(shifted, shift); - if (SIGN_VALUES) { - shifted = _mm_srai_epi16(shifted, shift); - } else { - shifted = _mm_srli_epi16(shifted, shift); - } + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); + const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); + shifted = _mm_or_si128(shifted, shifted2); + } - return shifted; - } + constexpr u32 shift = 16 - BIT_WIDTH; + shifted = _mm_slli_epi16(shifted, shift); + if (SIGN_VALUES) { + shifted = _mm_srai_epi16(shifted, shift); + } else { + shifted = _mm_srli_epi16(shifted, shift); } - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i &input) { - using tables = unpack_tables_avx512_24; + return shifted; + } +} - const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute(), input); +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i& input) { + using tables = unpack_tables_avx512_24; - constexpr u32 shift = 32 - BIT_WIDTH; - __m128i shifted = _mm_sllv_epi32(permuted, tables::get_shift()); - if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { - shifted = _mm_srai_epi32(shifted, shift); - } else { - shifted = _mm_srli_epi32(shifted, shift); - } + const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute(), input); - return shifted; - } - } // namespace m128 + constexpr u32 shift = 32 - BIT_WIDTH; + __m128i shifted = _mm_sllv_epi32(permuted, tables::get_shift()); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + shifted = _mm_srai_epi32(shifted, shift); + } else { + shifted = _mm_srli_epi32(shifted, shift); + } - namespace m256 { - constexpr __mmask32 kAlternateByteMask32 = 0xAAAAAAAAULL; + return shifted; +} +} // namespace m128 + +namespace m256 { +constexpr __mmask32 kAlternateByteMask32 = 0xAAAAAAAAULL; __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0x00ff); - const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); - const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); - return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); - } + const __m256i mask = _mm256_set1_epi16(0x00ff); + const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); + const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); + return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); +} __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0xff00); - const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); - const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); - return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); - } + const __m256i mask = _mm256_set1_epi16(0xff00); + const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); + const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); + return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); +} __always_inline static __m256i _mm256_slli_epi8(const __m256i a, const i8 imm8) { - return _mm256_sllv_epi8(a, _mm256_set1_epi8(imm8)); - } + return _mm256_sllv_epi8(a, _mm256_set1_epi8(imm8)); +} __always_inline static __m256i _mm256_srli_epi8(const __m256i a, const i8 imm8) { - const __m256i lo_mask = _mm256_set1_epi16(0x00ff); - const __m256i hi_mask = _mm256_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); + const __m256i lo_mask = _mm256_set1_epi16(0x00ff); + const __m256i hi_mask = _mm256_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); - const __m256i lo = _mm256_srl_epi16(_mm256_and_si256(a, lo_mask), shift); - const __m256i hi = _mm256_and_si256(_mm256_srl_epi16(a, shift), hi_mask); + const __m256i lo = _mm256_srl_epi16(_mm256_and_si256(a, lo_mask), shift); + const __m256i hi = _mm256_and_si256(_mm256_srl_epi16(a, shift), hi_mask); - return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); - } + return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); +} __always_inline static __m256i _mm256_srai_epi8(const __m256i a, const i8 imm8) { - const __m256i lo_mask = _mm256_set1_epi16(0x00ff); - const __m256i hi_mask = _mm256_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); - - const __m256i hi = _mm256_and_si256(_mm256_sra_epi16(a, shift), hi_mask); - - const __m256i lo_as_hi = _mm256_slli_epi16(_mm256_and_si256(a, lo_mask), 8); - const __m256i lo = _mm256_and_si256(_mm256_srli_epi16(_mm256_sra_epi16(lo_as_hi, shift), 8), lo_mask); - - return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); - } - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i &input) { - if constexpr (BIT_WIDTH == 8) { - return input; + const __m256i lo_mask = _mm256_set1_epi16(0x00ff); + const __m256i hi_mask = _mm256_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); + + const __m256i hi = _mm256_and_si256(_mm256_sra_epi16(a, shift), hi_mask); + + const __m256i lo_as_hi = _mm256_slli_epi16(_mm256_and_si256(a, lo_mask), 8); + const __m256i lo = _mm256_and_si256(_mm256_srli_epi16(_mm256_sra_epi16(lo_as_hi, shift), 8), lo_mask); + + return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i& input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + if constexpr (BIT_WIDTH == 1) { + const auto value = static_cast<__mmask32>(_mm_cvtsi128_si64(_mm256_castsi256_si128(input))); + const __m256i source = _mm256_movm_epi8(value); + const __m256i unpacked = _mm256_abs_epi8(source); + return unpacked; + } else if constexpr (BIT_WIDTH == 2) { + __m256i values_shift0 = input; + __m256i values_shift2 = _mm256_srli_epi16(values_shift0, 2); + const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); + const __m256i values_shift6 = _mm256_srli_epi16(values_shift0, 6); + + __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift2); + values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift2); + values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); + + interleave_tmp = _mm256_unpacklo_epi8(values_shift4, values_shift6); + values_shift2 = _mm256_unpackhi_epi8(values_shift4, values_shift6); + values_shift2 = _mm256_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); + + interleave_tmp = _mm256_unpacklo_epi16(values_shift0, values_shift2); + values_shift0 = _mm256_unpackhi_epi16(values_shift0, values_shift2); + values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); + values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); + + values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0303)); + + return values_shift0; + } else if constexpr (BIT_WIDTH == 4) { + __m256i values_shift0 = input; + const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); + + __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift4); + values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift4); + values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); + values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); + + values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0F0F)); + + return values_shift0; + } else { + using tables = unpack_tables_avx512_8; + + const __m256i permuted1 = _mm256_permutexvar_epi8(tables::get_permute1(), input); + const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); + + const __m256i shifted1 = _mm256_srlv_epi8(permuted1, tables::get_shift1()); + const __m256i shifted2 = _mm256_sllv_epi8(permuted2, tables::get_shift2()); + + const __mmask32 spill_mask = _mm256_cmpneq_epi8_mask(tables::get_shift2(), _mm256_setzero_si256()); + __m256i combined = _mm256_or_si256(shifted1, _mm256_maskz_mov_epi8(spill_mask, shifted2)); + + constexpr u32 shift = 8 - BIT_WIDTH; + combined = _mm256_slli_epi8(combined, shift); + if (SIGN_VALUES) { + combined = _mm256_srai_epi8(combined, shift); } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask32>(_mm_cvtsi128_si64(_mm256_castsi256_si128(input))); - const __m256i source = _mm256_movm_epi8(value); - const __m256i unpacked = _mm256_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m256i values_shift0 = input; - __m256i values_shift2 = _mm256_srli_epi16(values_shift0, 2); - const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); - const __m256i values_shift6 = _mm256_srli_epi16(values_shift0, 6); - - __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); - - interleave_tmp = _mm256_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm256_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm256_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); - - interleave_tmp = _mm256_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm256_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); - values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); - - values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0303)); - - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m256i values_shift0 = input; - const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); - - __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); - values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); - - values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0F0F)); - - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m256i permuted1 = _mm256_permutexvar_epi8(tables::get_permute1(), input); - const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); - - const __m256i shifted1 = _mm256_srlv_epi8(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask32 spill_mask = _mm256_cmpneq_epi8_mask(tables::get_shift2(), _mm256_setzero_si256()); - __m256i combined = _mm256_or_si256(shifted1, _mm256_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr u32 shift = 8 - BIT_WIDTH; - combined = _mm256_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm256_srai_epi8(combined, shift); - } else { - combined = _mm256_srli_epi8(combined, shift); - } - - return combined; - } + combined = _mm256_srli_epi8(combined, shift); } - } - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i &input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = unpack_tables_avx512_16; + return combined; + } + } +} - const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute1(), input); +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i& input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = unpack_tables_avx512_16; - __m256i shifted = _mm256_srlv_epi16(permuted, tables::get_shift1()); + const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute1(), input); - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); - const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); - shifted = _mm256_or_si256(shifted, shifted2); - } + __m256i shifted = _mm256_srlv_epi16(permuted, tables::get_shift1()); - constexpr u32 shift = 16 - BIT_WIDTH; - shifted = _mm256_slli_epi16(shifted, shift); - if (SIGN_VALUES) { - shifted = _mm256_srai_epi16(shifted, shift); - } else { - shifted = _mm256_srli_epi16(shifted, shift); - } + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); + const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); + shifted = _mm256_or_si256(shifted, shifted2); + } - return shifted; - } + constexpr u32 shift = 16 - BIT_WIDTH; + shifted = _mm256_slli_epi16(shifted, shift); + if (SIGN_VALUES) { + shifted = _mm256_srai_epi16(shifted, shift); + } else { + shifted = _mm256_srli_epi16(shifted, shift); } - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i &input) { - using tables = unpack_tables_avx512_24; + return shifted; + } +} - const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute(), input); +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i& input) { + using tables = unpack_tables_avx512_24; - constexpr u32 shift = 32 - BIT_WIDTH; - __m256i shifted = _mm256_sllv_epi32(permuted, tables::get_shift()); - if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { - shifted = _mm256_srai_epi32(shifted, shift); - } else { - shifted = _mm256_srli_epi32(shifted, shift); - } + const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute(), input); - return shifted; - } - } // namespace m256 + constexpr u32 shift = 32 - BIT_WIDTH; + __m256i shifted = _mm256_sllv_epi32(permuted, tables::get_shift()); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + shifted = _mm256_srai_epi32(shifted, shift); + } else { + shifted = _mm256_srli_epi32(shifted, shift); + } - namespace m512 { - constexpr __mmask64 kAlternateByteMask64 = 0xAAAAAAAAAAAAAAAAULL; + return shifted; +} +} // namespace m256 + +namespace m512 { +constexpr __mmask64 kAlternateByteMask64 = 0xAAAAAAAAAAAAAAAAULL; __always_inline static __m512i _mm512_srlv_epi8(const __m512i a, const __m512i count) { - const __m512i mask = _mm512_set1_epi16(0x00ff); - const __m512i low_half = _mm512_srlv_epi16(_mm512_and_si512(mask, a), _mm512_and_si512(mask, count)); - const __m512i high_half = _mm512_srlv_epi16(a, _mm512_srli_epi16(count, 8)); - return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); - } + const __m512i mask = _mm512_set1_epi16(0x00ff); + const __m512i low_half = _mm512_srlv_epi16(_mm512_and_si512(mask, a), _mm512_and_si512(mask, count)); + const __m512i high_half = _mm512_srlv_epi16(a, _mm512_srli_epi16(count, 8)); + return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); +} __always_inline static __m512i _mm512_sllv_epi8(const __m512i a, const __m512i count) { - const __m512i mask = _mm512_set1_epi16(0xff00); - const __m512i low_half = _mm512_sllv_epi16(a, _mm512_andnot_si512(mask, count)); - const __m512i high_half = _mm512_sllv_epi16(_mm512_and_si512(mask, a), _mm512_srli_epi16(count, 8)); - return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); - } + const __m512i mask = _mm512_set1_epi16(0xff00); + const __m512i low_half = _mm512_sllv_epi16(a, _mm512_andnot_si512(mask, count)); + const __m512i high_half = _mm512_sllv_epi16(_mm512_and_si512(mask, a), _mm512_srli_epi16(count, 8)); + return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); +} __always_inline static __m512i _mm512_slli_epi8(const __m512i a, const i8 imm8) { - return _mm512_sllv_epi8(a, _mm512_set1_epi8(imm8)); - } + return _mm512_sllv_epi8(a, _mm512_set1_epi8(imm8)); +} __always_inline static __m512i _mm512_srli_epi8(const __m512i a, const i8 imm8) { - const __m512i lo_mask = _mm512_set1_epi16(0x00ff); - const __m512i hi_mask = _mm512_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); + const __m512i lo_mask = _mm512_set1_epi16(0x00ff); + const __m512i hi_mask = _mm512_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); - const __m512i lo = _mm512_srl_epi16(_mm512_and_si512(a, lo_mask), shift); - const __m512i hi = _mm512_and_si512(_mm512_srl_epi16(a, shift), hi_mask); + const __m512i lo = _mm512_srl_epi16(_mm512_and_si512(a, lo_mask), shift); + const __m512i hi = _mm512_and_si512(_mm512_srl_epi16(a, shift), hi_mask); - return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); - } + return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); +} __always_inline static __m512i _mm512_srai_epi8(const __m512i a, const i8 imm8) { - const __m512i lo_mask = _mm512_set1_epi16(0x00ff); - const __m512i hi_mask = _mm512_set1_epi16(0xff00); - const __m128i shift = _mm_cvtsi32_si128(imm8); - - const __m512i hi = _mm512_and_si512(_mm512_sra_epi16(a, shift), hi_mask); - - const __m512i lo_as_hi = _mm512_slli_epi16(_mm512_and_si512(a, lo_mask), 8); - const __m512i lo = _mm512_and_si512(_mm512_srli_epi16(_mm512_sra_epi16(lo_as_hi, shift), 8), lo_mask); - - return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); - } - - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i &input) { - if constexpr (BIT_WIDTH == 8) { - return input; + const __m512i lo_mask = _mm512_set1_epi16(0x00ff); + const __m512i hi_mask = _mm512_set1_epi16(0xff00); + const __m128i shift = _mm_cvtsi32_si128(imm8); + + const __m512i hi = _mm512_and_si512(_mm512_sra_epi16(a, shift), hi_mask); + + const __m512i lo_as_hi = _mm512_slli_epi16(_mm512_and_si512(a, lo_mask), 8); + const __m512i lo = _mm512_and_si512(_mm512_srli_epi16(_mm512_sra_epi16(lo_as_hi, shift), 8), lo_mask); + + return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i& input) { + if constexpr (BIT_WIDTH == 8) { + return input; + } else { + if constexpr (BIT_WIDTH == 1) { + const auto value = static_cast<__mmask64>(_mm_cvtsi128_si64(_mm512_castsi512_si128(input))); + const __m512i source = _mm512_movm_epi8(value); + const __m512i unpacked = _mm512_abs_epi8(source); + return unpacked; + } else if constexpr (BIT_WIDTH == 2) { + __m512i values_shift0 = input; + __m512i values_shift2 = _mm512_srli_epi16(values_shift0, 2); + const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); + const __m512i values_shift6 = _mm512_srli_epi16(values_shift0, 6); + + __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift2); + values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift2); + values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); + + interleave_tmp = _mm512_unpacklo_epi8(values_shift4, values_shift6); + values_shift2 = _mm512_unpackhi_epi8(values_shift4, values_shift6); + values_shift2 = _mm512_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); + + interleave_tmp = _mm512_unpacklo_epi16(values_shift0, values_shift2); + values_shift0 = _mm512_unpackhi_epi16(values_shift0, values_shift2); + values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x88); + values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); + + values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0303)); + + return values_shift0; + } else if constexpr (BIT_WIDTH == 4) { + __m512i values_shift0 = input; + const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); + + __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift4); + values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift4); + values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x44); + values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); + + values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0F0F)); + + return values_shift0; + } else { + using tables = unpack_tables_avx512_8; + + const __m512i permuted1 = _mm512_permutexvar_epi8(tables::get_permute1(), input); + const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); + + const __m512i shifted1 = _mm512_srlv_epi8(permuted1, tables::get_shift1()); + const __m512i shifted2 = _mm512_sllv_epi8(permuted2, tables::get_shift2()); + + const __mmask64 spill_mask = _mm512_cmpneq_epi8_mask(tables::get_shift2(), _mm512_setzero_si512()); + __m512i combined = _mm512_or_si512(shifted1, _mm512_maskz_mov_epi8(spill_mask, shifted2)); + + constexpr u32 shift = 8 - BIT_WIDTH; + combined = _mm512_slli_epi8(combined, shift); + if (SIGN_VALUES) { + combined = _mm512_srai_epi8(combined, shift); } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask64>(_mm_cvtsi128_si64(_mm512_castsi512_si128(input))); - const __m512i source = _mm512_movm_epi8(value); - const __m512i unpacked = _mm512_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m512i values_shift0 = input; - __m512i values_shift2 = _mm512_srli_epi16(values_shift0, 2); - const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); - const __m512i values_shift6 = _mm512_srli_epi16(values_shift0, 6); - - __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); - - interleave_tmp = _mm512_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm512_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm512_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); - - interleave_tmp = _mm512_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm512_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x88); - values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); - - values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0303)); - - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m512i values_shift0 = input; - const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); - - __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x44); - values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); - - values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0F0F)); - - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m512i permuted1 = _mm512_permutexvar_epi8(tables::get_permute1(), input); - const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); - - const __m512i shifted1 = _mm512_srlv_epi8(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask64 spill_mask = _mm512_cmpneq_epi8_mask(tables::get_shift2(), _mm512_setzero_si512()); - __m512i combined = _mm512_or_si512(shifted1, _mm512_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr u32 shift = 8 - BIT_WIDTH; - combined = _mm512_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm512_srai_epi8(combined, shift); - } else { - combined = _mm512_srli_epi8(combined, shift); - } - - return combined; - } + combined = _mm512_srli_epi8(combined, shift); } + + return combined; + } + } +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i& input) { + if constexpr (BIT_WIDTH == 16) { + return input; + } else { + using tables = unpack_tables_avx512_16; + + const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute1(), input); + __m512i shifted = _mm512_srlv_epi16(permuted, tables::get_shift1()); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); + const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); + shifted = _mm512_or_si512(shifted, shifted2); } - template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i &input) { - if constexpr (BIT_WIDTH == 16) { - return input; - } else { - using tables = unpack_tables_avx512_16; - - const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute1(), input); - __m512i shifted = _mm512_srlv_epi16(permuted, tables::get_shift1()); - - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); - const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); - shifted = _mm512_or_si512(shifted, shifted2); - } - - constexpr u32 shift = 16 - BIT_WIDTH; - shifted = _mm512_slli_epi16(shifted, shift); - if (SIGN_VALUES) { - shifted = _mm512_srai_epi16(shifted, shift); - } else { - shifted = _mm512_srli_epi16(shifted, shift); - } - - return shifted; - } + constexpr u32 shift = 16 - BIT_WIDTH; + shifted = _mm512_slli_epi16(shifted, shift); + if (SIGN_VALUES) { + shifted = _mm512_srai_epi16(shifted, shift); + } else { + shifted = _mm512_srli_epi16(shifted, shift); } - template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i &input) { - using tables = unpack_tables_avx512_24; + return shifted; + } +} - const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute(), input); - __m512i shifted = _mm512_sllv_epi32(permuted, tables::get_shift()); +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i& input) { + using tables = unpack_tables_avx512_24; - constexpr u32 shift = 32 - BIT_WIDTH; - if constexpr (SIGN_VALUES) { - shifted = _mm512_srai_epi32(shifted, shift); - } else { - shifted = _mm512_srli_epi32(shifted, shift); - } + const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute(), input); + __m512i shifted = _mm512_sllv_epi32(permuted, tables::get_shift()); - return shifted; - } - } // namespace m512 + constexpr u32 shift = 32 - BIT_WIDTH; + if constexpr (SIGN_VALUES) { + shifted = _mm512_srai_epi32(shifted, shift); + } else { + shifted = _mm512_srli_epi32(shifted, shift); + } + + return shifted; +} +} // namespace m512 } // namespace pernix::internal #endif // PERNIX_AVX512VBMI_UNPACKING_H diff --git a/include/pernix/x86/bmi2/bmi2_compression.h b/include/pernix/x86/bmi2/bmi2_compression.h index 8a0cc57..b1f48da 100644 --- a/include/pernix/x86/bmi2/bmi2_compression.h +++ b/include/pernix/x86/bmi2/bmi2_compression.h @@ -9,148 +9,149 @@ #include #include + namespace pernix { - namespace internal { - /** +namespace internal { +/** * @brief Build the masks and shift constants used by the BMI2 packers. * * @tparam BIT_WIDTH bit width per packed value. * @return std::tuple mask tuple used by the BMI2 helpers. */ - template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) - static constexpr std::tuple pack_avx2_bmi2_constants() { - u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - u64 pext_mask; - u16 shift1 = BIT_WIDTH * 4; - u16 shift2 = 64 - shift1; - - if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { - pext_mask = 0x0101010101010101ULL * mask; - } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { - pext_mask = 0x0001000100010001ULL * mask; - } else { - pext_mask = 0x0000000100000001ULL * mask; - } +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) +static constexpr std::tuple pack_avx2_bmi2_constants() { + u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + u64 pext_mask; + u16 shift1 = BIT_WIDTH * 4; + u16 shift2 = 64 - shift1; + + if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { + pext_mask = 0x0101010101010101ULL * mask; + } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { + pext_mask = 0x0001000100010001ULL * mask; + } else { + pext_mask = 0x0000000100000001ULL * mask; + } - return { - mask, - pext_mask, - shift1, - shift2, - }; - } + return { + mask, + pext_mask, + shift1, + shift2, + }; +} - /** +/** * @brief Pack four 32-bit values with BMI2 extract instructions. * * @tparam BIT_WIDTH bit width per packed value. * @param input SIMD register containing four quantized values. * @return __m128i packed bitstream in the low bytes of the result. */ - template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) - static inline auto mm_pack_epi32_bmi2(const __m128i &input) -> __m128i { - const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); - - if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 16) { - const __m128i packed = _mm_packs_epi32(input, _mm_setzero_si128()); - const u64 value = _pext_u64(_mm_extract_epi64(packed, 0), pext_mask); - - const __m128i result = _mm_set_epi64x(0, value); - return result; - } else { - alignas(16) u64 values[2]; - values[0] = _pext_u64(_mm_extract_epi64(input, 0), pext_mask); - - const u64 temp_combined = _pext_u64(_mm_extract_epi64(input, 1), pext_mask); - values[1] = temp_combined >> shift2; - values[0] |= (temp_combined << shift1); - - const __m128i result = _mm_set_epi64x(static_cast(values[1]), static_cast(values[0])); - return result; - } - } +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) +static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { + const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); + + if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 16) { + const __m128i packed = _mm_packs_epi32(input, _mm_setzero_si128()); + const u64 value = _pext_u64(_mm_extract_epi64(packed, 0), pext_mask); + + const __m128i result = _mm_set_epi64x(0, value); + return result; + } else { + alignas(16) u64 values[2]; + values[0] = _pext_u64(_mm_extract_epi64(input, 0), pext_mask); + + const u64 temp_combined = _pext_u64(_mm_extract_epi64(input, 1), pext_mask); + values[1] = temp_combined >> shift2; + values[0] |= (temp_combined << shift1); + + const __m128i result = _mm_set_epi64x(static_cast(values[1]), static_cast(values[0])); + return result; + } +} - /** +/** * @brief Pack eight 32-bit values with BMI2 extract instructions. * * @tparam BIT_WIDTH bit width per packed value. * @param input SIMD register containing eight quantized values. * @return __m256i packed bitstream in the low bytes of the result. */ - template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) - static inline auto mm256_pack_epi32_bmi2(const __m256i &input) -> __m256i { - const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); - - if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { - const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); - const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); - const __m256i packed8 = _mm256_packs_epi16(permuted, _mm256_setzero_si256()); - const u64 value = _pext_u64(_mm256_extract_epi64(packed8, 0), pext_mask); - - const __m256i result = _mm256_setr_epi64x(static_cast(value), 0, 0, 0); - return result; - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); - alignas(16) i64 values[2] = {}; - values[0] = _pext_u64(_mm256_extract_epi64(packed16, 0), pext_mask); - - const u64 temp_combined = _pext_u64(_mm256_extract_epi64(packed16, 2), pext_mask); - values[1] = temp_combined >> shift2; - if constexpr (BIT_WIDTH != 16) { - values[0] |= static_cast(temp_combined << shift1); - } +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { + const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); + + if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { + const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); + const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); + const __m256i packed8 = _mm256_packs_epi16(permuted, _mm256_setzero_si256()); + const u64 value = _pext_u64(_mm256_extract_epi64(packed8, 0), pext_mask); + + const __m256i result = _mm256_setr_epi64x(static_cast(value), 0, 0, 0); + return result; + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); + alignas(16) i64 values[2] = {}; + values[0] = _pext_u64(_mm256_extract_epi64(packed16, 0), pext_mask); + + const u64 temp_combined = _pext_u64(_mm256_extract_epi64(packed16, 2), pext_mask); + values[1] = temp_combined >> shift2; + if constexpr (BIT_WIDTH != 16) { + values[0] |= static_cast(temp_combined << shift1); + } + + const __m256i result = _mm256_setr_epi64x(values[0], values[1], 0, 0); + return result; + } else { + constexpr u32 chunk_bits = BIT_WIDTH * 2; // bits extracted per 64-bit lane + static_assert(chunk_bits < 64); - const __m256i result = _mm256_setr_epi64x(values[0], values[1], 0, 0); - return result; + constexpr u64 chunk_mask = (chunk_bits == 64) ? ~u64{0} : ((u64{1} << chunk_bits) - 1); + + const u64 x0 = _pext_u64(_mm256_extract_epi64(input, 0), pext_mask) & chunk_mask; + const u64 x1 = _pext_u64(_mm256_extract_epi64(input, 1), pext_mask) & chunk_mask; + const u64 x2 = _pext_u64(_mm256_extract_epi64(input, 2), pext_mask) & chunk_mask; + const u64 x3 = _pext_u64(_mm256_extract_epi64(input, 3), pext_mask) & chunk_mask; + + u64 out0 = 0; + u64 out1 = 0; + u64 out2 = 0; + + auto append_bits = [&](u64 value, u32 bit_offset) { + const u32 word = bit_offset >> 6; // / 64 + const u32 off = bit_offset & 63; // % 64 + + if (word == 0) { + out0 |= value << off; + if (off + chunk_bits > 64) { + out1 |= value >> (64 - off); + } + } else if (word == 1) { + out1 |= value << off; + if (off + chunk_bits > 64) { + out2 |= value >> (64 - off); + } } else { - constexpr u32 chunk_bits = BIT_WIDTH * 2; // bits extracted per 64-bit lane - static_assert(chunk_bits < 64); - - constexpr u64 chunk_mask = (chunk_bits == 64) ? ~u64{0} : ((u64{1} << chunk_bits) - 1); - - const u64 x0 = _pext_u64(_mm256_extract_epi64(input, 0), pext_mask) & chunk_mask; - const u64 x1 = _pext_u64(_mm256_extract_epi64(input, 1), pext_mask) & chunk_mask; - const u64 x2 = _pext_u64(_mm256_extract_epi64(input, 2), pext_mask) & chunk_mask; - const u64 x3 = _pext_u64(_mm256_extract_epi64(input, 3), pext_mask) & chunk_mask; - - u64 out0 = 0; - u64 out1 = 0; - u64 out2 = 0; - - auto append_bits = [&](u64 value, u32 bit_offset) { - const u32 word = bit_offset >> 6; // / 64 - const u32 off = bit_offset & 63; // % 64 - - if (word == 0) { - out0 |= value << off; - if (off + chunk_bits > 64) { - out1 |= value >> (64 - off); - } - } else if (word == 1) { - out1 |= value << off; - if (off + chunk_bits > 64) { - out2 |= value >> (64 - off); - } - } else { - out2 |= value << off; - } - }; - - append_bits(x0, 0 * chunk_bits); - append_bits(x1, 1 * chunk_bits); - append_bits(x2, 2 * chunk_bits); - append_bits(x3, 3 * chunk_bits); - - return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), - static_cast(out2), 0); + out2 |= value << off; } - } - } // namespace internal + }; + + append_bits(x0, 0 * chunk_bits); + append_bits(x1, 1 * chunk_bits); + append_bits(x2, 2 * chunk_bits); + append_bits(x3, 3 * chunk_bits); - /** + return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), + static_cast(out2), 0); + } +} +} // namespace internal + +/** * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -162,47 +163,47 @@ namespace pernix { * * @note This function requires AVX2 and BMI2 support. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_compress_block_bmi2(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 iterations_8 = elements_per_block / 8; - constexpr u8 remaining = elements_per_block - iterations_8 * 8; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - const __m256 scale_v = _mm256_set1_ps(scale); + const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 4 - for (u32 iter = 0; iter < iterations_8; iter++) { - const __m256 source = _mm256_loadu_ps(input); - const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); - const __m256i packed_input = internal::mm256_clamp_signed_epi32 < BIT_WIDTH > (quantized); - const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); - std::memcpy(output, &packed, BIT_WIDTH); - input += 8; - output += BIT_WIDTH; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256 source = _mm256_loadu_ps(input); + const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); + const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); + std::memcpy(output, &packed, BIT_WIDTH); + input += 8; + output += BIT_WIDTH; + } - if constexpr (remaining) { - std::vector block_values(remaining); + if constexpr (remaining) { + std::vector block_values(remaining); #pragma GCC unroll 8 - for (u32 i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized < BIT_WIDTH > ( - internal::quantize_ps_epi32(input[i], scale))); - } - - internal::pack_epi32_fallback < BIT_WIDTH > (block_values, output); + for (u32 i = 0; i < remaining; i++) { + block_values[i] = + static_cast(internal::clamp_signed_quantized( + internal::quantize_ps_epi32(input[i], scale))); } - return 0; + internal::pack_epi32_fallback(block_values, output); } - /** + return 0; +} + +/** * @brief Compress a single block of double values using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -213,50 +214,50 @@ namespace pernix { * * @note This function requires AVX2 and BMI2 support. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_compress_block_bmi2(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 iterations_8 = elements_per_block / 8; - constexpr u8 remaining = elements_per_block - iterations_8 * 8; + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; - std::memset(output, 0, BLOCK_SIZE); + std::memset(output, 0, BLOCK_SIZE); - const __m256d scale_v = _mm256_set1_pd(scale); + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 4 - for (u32 iter = 0; iter < iterations_8; iter++) { - const __m256d source1 = _mm256_loadu_pd(input); - const __m256d source2 = _mm256_loadu_pd(input + 4); - const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); - const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); - __m256i combined = _mm256_castsi128_si256(quantized1); - combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_bmi2( - internal::mm256_clamp_signed_epi32 < BIT_WIDTH > (combined)); - std::memcpy(output, &packed, BIT_WIDTH); - input += 8; - output += BIT_WIDTH; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256d source1 = _mm256_loadu_pd(input); + const __m256d source2 = _mm256_loadu_pd(input + 4); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed = internal::mm256_pack_epi32_bmi2( + internal::mm256_clamp_signed_epi32(combined)); + std::memcpy(output, &packed, BIT_WIDTH); + input += 8; + output += BIT_WIDTH; + } - if constexpr (remaining) { - std::vector block_values(remaining); + if constexpr (remaining) { + std::vector block_values(remaining); #pragma GCC unroll 8 - for (u32 i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized < BIT_WIDTH > ( - internal::quantize_pd_epi64(input[i], scale))); - } - - internal::pack_epi32_fallback < BIT_WIDTH > (block_values, output); + for (u32 i = 0; i < remaining; i++) { + block_values[i] = + static_cast(internal::clamp_signed_quantized( + internal::quantize_pd_epi64(input[i], scale))); } - return 0; + + internal::pack_epi32_fallback(block_values, output); } + return 0; +} - /** +/** * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -269,26 +270,26 @@ namespace pernix { * * @note This function requires AVX2 and BMI2 support. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_compress_blocks_bmi2(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr, const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const f32 *block_input = input; - u8 *block_output = output; - - for (u32 block = 0; block < blocks; block++) { - mm256_compress_block_bmi2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - - return 0; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f32* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_bmi2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; } - /** + return 0; +} + +/** * @brief Compress multiple blocks of double values using AVX2 and BMI2 instructions. * * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). @@ -300,24 +301,24 @@ namespace pernix { * * @note This function requires AVX2 and BMI2 support. */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_compress_blocks_bmi2(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr, const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const f64 *block_input = input; - u8 *block_output = output; - - for (u32 block = 0; block < blocks; block++) { - mm256_compress_block_bmi2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - - return 0; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f64* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_bmi2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; } + + return 0; +} } // namespace pernix #endif // PERNIX_BMI2_COMPRESSION_H diff --git a/include/pernix/x86/bmi2/bmi2_decompression.h b/include/pernix/x86/bmi2/bmi2_decompression.h index 275a11a..69bef94 100644 --- a/include/pernix/x86/bmi2/bmi2_decompression.h +++ b/include/pernix/x86/bmi2/bmi2_decompression.h @@ -9,359 +9,360 @@ #include #include + namespace pernix { - namespace internal { - /** - * @brief Sign-extend packed values after BMI2 expansion into 32-bit lanes. - * - * @tparam BIT_WIDTH original encoded bit width. - * @param source register containing unpacked values. - * @return __m128i sign-extended values. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - __m128i mm_sign_extend32(__m128i source) { - if constexpr (BIT_WIDTH == 1) { - // Keep 1-bit values as 0/1 to match fallback decoding semantics. - return source; - } - - constexpr u16 shift = 32 - BIT_WIDTH; - source = _mm_slli_epi32(source, shift); - return _mm_srai_epi32(source, shift); - } +namespace internal { +/** +* @brief Sign-extend packed values after BMI2 expansion into 32-bit lanes. +* +* @tparam BIT_WIDTH original encoded bit width. +* @param source register containing unpacked values. +* @return __m128i sign-extended values. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__m128i mm_sign_extend32(__m128i source) { + if constexpr (BIT_WIDTH == 1) { + // Keep 1-bit values as 0/1 to match fallback decoding semantics. + return source; + } - /** - * @brief Sign-extend packed values after BMI2 expansion into eight 32-bit lanes. - * - * @tparam BIT_WIDTH original encoded bit width. - * @param source register containing unpacked values. - * @return __m256i sign-extended values. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - __m256i mm256_sign_extend32(__m256i source) { - if constexpr (BIT_WIDTH == 1) { - // Keep 1-bit values as 0/1 to match fallback decoding semantics. - return source; - } - - constexpr u16 shift = 32 - BIT_WIDTH; - source = _mm256_slli_epi32(source, shift); - return _mm256_srai_epi32(source, shift); - } + constexpr u16 shift = 32 - BIT_WIDTH; + source = _mm_slli_epi32(source, shift); + return _mm_srai_epi32(source, shift); +} + +/** +* @brief Sign-extend packed values after BMI2 expansion into eight 32-bit lanes. +* +* @tparam BIT_WIDTH original encoded bit width. +* @param source register containing unpacked values. +* @return __m256i sign-extended values. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__m256i mm256_sign_extend32(__m256i source) { + if constexpr (BIT_WIDTH == 1) { + // Keep 1-bit values as 0/1 to match fallback decoding semantics. + return source; + } - /** - * @brief Unpack four values from a BMI2-packed input buffer. - * - * @tparam BIT_WIDTH bit width per packed value. - * @tparam SIGN_VALUES whether to sign-extend the unpacked values. - * @param input pointer to the packed input buffer. - * @return __m128i unpacked values. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH > 0 && BIT_WIDTH <= 24) - __m128i mm_unpack_epi32_bmi2(const u8 * __restrict__ input) { - constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; - - __m128i result; - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - constexpr u64 pdep_mask = 0x0101010101010101ULL * mask; - - u32 temp_value = 0; - std::memcpy(&temp_value, input, packed_bytes); - - const i32 value = _pdep_u32(temp_value, static_cast(pdep_mask)); - const __m128i source = _mm_insert_epi32(_mm_setzero_si128(), value, 0); - - result = _mm_cvtepi8_epi32(source); - } else if constexpr (BIT_WIDTH == 16) { - const __m128i source = _mm_loadu_si64(input); - result = _mm_cvtepi16_epi32(source); - } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { - constexpr u64 pdep_mask = 0x0001000100010001ULL * mask; - - u64 temp_value = 0; - std::memcpy(&temp_value, input, packed_bytes); - - const i64 value = _pdep_u64(temp_value, pdep_mask); - const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); - - result = _mm_cvtepi16_epi32(source); - } else { - constexpr u64 pdep_mask = 0x0000000100000001ULL * mask; - constexpr u32 shift1 = BIT_WIDTH * 2; - constexpr u32 shift2 = 64 - shift1; - - alignas(16) u64 temp_values[2]{}; - std::memcpy(temp_values, input, packed_bytes); - - alignas(16) i64 values[2]; - values[0] = _pdep_u64(temp_values[0], pdep_mask); - values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); - - result = _mm_set_epi64x(values[1], values[0]); - } - - if constexpr (SIGN_VALUES) { - result = internal::mm_sign_extend32(result); - } - return result; - } + constexpr u16 shift = 32 - BIT_WIDTH; + source = _mm256_slli_epi32(source, shift); + return _mm256_srai_epi32(source, shift); +} + +/** +* @brief Unpack four values from a BMI2-packed input buffer. +* +* @tparam BIT_WIDTH bit width per packed value. +* @tparam SIGN_VALUES whether to sign-extend the unpacked values. +* @param input pointer to the packed input buffer. +* @return __m128i unpacked values. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH > 0 && BIT_WIDTH <= 24) +__m128i mm_unpack_epi32_bmi2(const u8* __restrict__ input) { + constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; + + __m128i result; + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + constexpr u64 pdep_mask = 0x0101010101010101ULL * mask; + + u32 temp_value = 0; + std::memcpy(&temp_value, input, packed_bytes); + + const i32 value = _pdep_u32(temp_value, static_cast(pdep_mask)); + const __m128i source = _mm_insert_epi32(_mm_setzero_si128(), value, 0); + + result = _mm_cvtepi8_epi32(source); + } else if constexpr (BIT_WIDTH == 16) { + const __m128i source = _mm_loadu_si64(input); + result = _mm_cvtepi16_epi32(source); + } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { + constexpr u64 pdep_mask = 0x0001000100010001ULL * mask; + + u64 temp_value = 0; + std::memcpy(&temp_value, input, packed_bytes); + + const i64 value = _pdep_u64(temp_value, pdep_mask); + const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); + + result = _mm_cvtepi16_epi32(source); + } else { + constexpr u64 pdep_mask = 0x0000000100000001ULL * mask; + constexpr u32 shift1 = BIT_WIDTH * 2; + constexpr u32 shift2 = 64 - shift1; + + alignas(16) u64 temp_values[2]{}; + std::memcpy(temp_values, input, packed_bytes); + + alignas(16) i64 values[2]; + values[0] = _pdep_u64(temp_values[0], pdep_mask); + values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); + + result = _mm_set_epi64x(values[1], values[0]); + } - /** - * @brief Unpack eight values from a BMI2-packed input buffer. - * - * @tparam BIT_WIDTH bit width per packed value. - * @tparam SIGN_VALUES whether to sign-extend the unpacked values. - * @param input pointer to the packed input buffer. - * @return __m256i unpacked values. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) - __m256i mm256_unpack_epi32_bmi2(const u8 * __restrict__ input) { - constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - constexpr std::size_t packed_bytes = BIT_WIDTH; - - __m256i result; - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - constexpr u64 pdep_mask = 0x0101010101010101ULL * mask; - - u64 temp_value = 0; - std::memcpy(&temp_value, input, packed_bytes); - - const i64 value = _pdep_u64(temp_value, pdep_mask); - const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); - - result = _mm256_cvtepi8_epi32(source); - } else if constexpr (BIT_WIDTH == 16) { - const __m128i source = _mm_loadu_si128(reinterpret_cast(input)); - result = _mm256_cvtepi16_epi32(source); - } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { - constexpr u64 pdep_mask = 0x0001000100010001ULL * mask; - constexpr u64 shift1 = BIT_WIDTH * 4; - constexpr u64 shift2 = 64 - shift1; - - alignas(16) u64 temp_values[2]{}; - std::memcpy(temp_values, input, packed_bytes); - - alignas(16) i64 values[2]; - values[0] = _pdep_u64(temp_values[0], pdep_mask); - values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); - - const __m128i source = _mm_set_epi64x(values[1], values[0]); - result = _mm256_cvtepi16_epi32(source); - } else { - constexpr u64 pdep_mask = 0x0000000100000001ULL * mask; - constexpr u32 shift1 = BIT_WIDTH * 2; - constexpr u32 shift2 = 64 - shift1; - - alignas(16) u64 temp_values[4]{}; - std::memcpy(temp_values, input, packed_bytes); - - if constexpr ((BIT_WIDTH % 2) == 0) { - std::memcpy(temp_values + 2, input + BIT_WIDTH / 2, packed_bytes - BIT_WIDTH / 2); - } else { - constexpr u32 second_group_bit_offset = BIT_WIDTH * 4; - constexpr u32 second_group_byte_offset = second_group_bit_offset / 8; - constexpr u32 second_group_shift = second_group_bit_offset % 8; - - alignas(16) u64 raw_values[2]{}; - std::memcpy(raw_values, input + second_group_byte_offset, packed_bytes - second_group_byte_offset); - - temp_values[2] = (raw_values[0] >> second_group_shift) | ( - raw_values[1] << (64 - second_group_shift)); - temp_values[3] = raw_values[1] >> second_group_shift; - } - - alignas(16) u64 values[4]; - values[0] = _pdep_u64((temp_values[0]), pdep_mask); - values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); - values[2] = _pdep_u64((temp_values[2]), pdep_mask); - values[3] = _pdep_u64((temp_values[2] >> shift1) | (temp_values[3] << shift2), pdep_mask); - - result = _mm256_set_epi64x(static_cast(values[3]), static_cast(values[2]), - static_cast(values[1]), - static_cast(values[0])); - } - - if constexpr (SIGN_VALUES) { - result = internal::mm256_sign_extend32(result); - } - return result; + if constexpr (SIGN_VALUES) { + result = internal::mm_sign_extend32(result); + } + return result; +} + +/** +* @brief Unpack eight values from a BMI2-packed input buffer. +* +* @tparam BIT_WIDTH bit width per packed value. +* @tparam SIGN_VALUES whether to sign-extend the unpacked values. +* @param input pointer to the packed input buffer. +* @return __m256i unpacked values. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__m256i mm256_unpack_epi32_bmi2(const u8* __restrict__ input) { + constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + constexpr std::size_t packed_bytes = BIT_WIDTH; + + __m256i result; + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + constexpr u64 pdep_mask = 0x0101010101010101ULL * mask; + + u64 temp_value = 0; + std::memcpy(&temp_value, input, packed_bytes); + + const i64 value = _pdep_u64(temp_value, pdep_mask); + const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); + + result = _mm256_cvtepi8_epi32(source); + } else if constexpr (BIT_WIDTH == 16) { + const __m128i source = _mm_loadu_si128(reinterpret_cast(input)); + result = _mm256_cvtepi16_epi32(source); + } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { + constexpr u64 pdep_mask = 0x0001000100010001ULL * mask; + constexpr u64 shift1 = BIT_WIDTH * 4; + constexpr u64 shift2 = 64 - shift1; + + alignas(16) u64 temp_values[2]{}; + std::memcpy(temp_values, input, packed_bytes); + + alignas(16) i64 values[2]; + values[0] = _pdep_u64(temp_values[0], pdep_mask); + values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); + + const __m128i source = _mm_set_epi64x(values[1], values[0]); + result = _mm256_cvtepi16_epi32(source); + } else { + constexpr u64 pdep_mask = 0x0000000100000001ULL * mask; + constexpr u32 shift1 = BIT_WIDTH * 2; + constexpr u32 shift2 = 64 - shift1; + + alignas(16) u64 temp_values[4]{}; + std::memcpy(temp_values, input, packed_bytes); + + if constexpr ((BIT_WIDTH % 2) == 0) { + std::memcpy(temp_values + 2, input + BIT_WIDTH / 2, packed_bytes - BIT_WIDTH / 2); + } else { + constexpr u32 second_group_bit_offset = BIT_WIDTH * 4; + constexpr u32 second_group_byte_offset = second_group_bit_offset / 8; + constexpr u32 second_group_shift = second_group_bit_offset % 8; + + alignas(16) u64 raw_values[2]{}; + std::memcpy(raw_values, input + second_group_byte_offset, packed_bytes - second_group_byte_offset); + + temp_values[2] = (raw_values[0] >> second_group_shift) | ( + raw_values[1] << (64 - second_group_shift)); + temp_values[3] = raw_values[1] >> second_group_shift; } - } // namespace internal - - /** - * @brief Decompress a single 512\-bit block using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_decompress_block_bmi2(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 iterations_8 = elements_per_block / 8; - constexpr u8 remaining = elements_per_block - iterations_8 * 8; - - const __m256 scale_v = _mm256_set1_ps(scale); + + alignas(16) u64 values[4]; + values[0] = _pdep_u64((temp_values[0]), pdep_mask); + values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); + values[2] = _pdep_u64((temp_values[2]), pdep_mask); + values[3] = _pdep_u64((temp_values[2] >> shift1) | (temp_values[3] << shift2), pdep_mask); + + result = _mm256_set_epi64x(static_cast(values[3]), static_cast(values[2]), + static_cast(values[1]), + static_cast(values[0])); + } + + if constexpr (SIGN_VALUES) { + result = internal::mm256_sign_extend32(result); + } + return result; +} +} // namespace internal + +/** +* @brief Decompress a single 512\-bit block using AVX2 and BMI2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). +* +* @param input pointer to the start of the compressed block. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed float values will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX2 and BMI2 support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + + const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 4 - for (u32 iter = 0; iter < iterations_8; iter++) { - const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); - const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); - _mm256_storeu_ps(output, dequantized); - input += BIT_WIDTH; - output += 8; - } + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); + const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); + _mm256_storeu_ps(output, dequantized); + input += BIT_WIDTH; + output += 8; + } - if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback < BIT_WIDTH, SIGN_VALUES + if constexpr (remaining > 0) { + const std::vector tail_values = internal::unpack_epi32_fallback (input, remaining); - for (u32 i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi32(tail_values[i], scale); - } + for (u32 i = 0; i < remaining; i++) { + output[i] = internal::dequantize_epi32(tail_values[i], scale); } - - return 0; } - /** - * @brief Decompress a single block to double values using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_decompress_block_bmi2(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 iterations_8 = elements_per_block / 8; - constexpr u8 remaining = elements_per_block - iterations_8 * 8; - const __m256d scale_v = _mm256_set1_pd(scale); + return 0; +} + +/** +* @brief Decompress a single block to double values using AVX2 and BMI2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). +* +* @param input pointer to the start of the compressed block. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed double values will be stored. +* @return int status code (0 for success). +* +* @note This function requires AVX2 and BMI2 support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 4 - for (u32 iter = 0; iter < iterations_8; iter++) { - const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); - const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); - const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); + const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); + const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); - const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); - const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); + const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); + const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); - _mm256_storeu_pd(output, dequantized1); - _mm256_storeu_pd(output + 4, dequantized2); + _mm256_storeu_pd(output, dequantized1); + _mm256_storeu_pd(output + 4, dequantized2); - input += BIT_WIDTH; - output += 8; - } + input += BIT_WIDTH; + output += 8; + } - if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback < BIT_WIDTH, SIGN_VALUES + if constexpr (remaining > 0) { + const std::vector tail_values = internal::unpack_epi32_fallback (input, remaining); - for (u32 i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi64(tail_values[i], scale); - } + for (u32 i = 0; i < remaining; i++) { + output[i] = internal::dequantize_epi64(tail_values[i], scale); } - - return 0; } - /** - * @brief Decompress multiple 512\-bit blocks using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_decompress_blocks_bmi2(const void * __restrict__ input_ptr, const f32 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const u8 *block_input = input; - f32 *block_output = output; - - for (u32 block = 0; block < blocks; block++) { - mm256_decompress_block_bmi2(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; + return 0; +} + +/** +* @brief Decompress multiple 512\-bit blocks using AVX2 and BMI2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). +* +* @param input pointer to the start of the compressed data. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed float values will be stored. +* @param blocks number of 512-bit blocks to decompress. +* @return int status code (0 for success). +* +* @note This function requires AVX2 and BMI2 support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const u8* block_input = input; + f32* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_decompress_block_bmi2(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } - /** - * @brief Decompress multiple blocks to double values using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @param blocks number of blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) - int mm256_decompress_blocks_bmi2(const void * __restrict__ input_ptr, const f64 scale, - void * __restrict__ output_ptr, - const u32 blocks) { - const auto *input = static_cast(input_ptr); - auto *output = static_cast(output_ptr); - - const u8 *block_input = input; - f64 *block_output = output; - - for (u32 block = 0; block < blocks; block++) { - mm256_decompress_block_bmi2(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; + return 0; +} + +/** +* @brief Decompress multiple blocks to double values using AVX2 and BMI2 instructions. +* +* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). +* @tparam SIGN_VALUES whether the values are signed or unsigned. +* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). +* +* @param input pointer to the start of the compressed data. +* @param scale scaling factor used during quantization. +* @param output pointer to the output buffer where decompressed double values will be stored. +* @param blocks number of blocks to decompress. +* @return int status code (0 for success). +* +* @note This function requires AVX2 and BMI2 support. +*/ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 scale, + void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const u8* block_input = input; + f64* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_decompress_block_bmi2(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } + + return 0; +} } // namespace pernix #endif // PERNIX_BMI2_DECOMPRESSION_H diff --git a/include/pernix/x86/utils.h b/include/pernix/x86/utils.h index 42aad76..2347eb6 100644 --- a/include/pernix/x86/utils.h +++ b/include/pernix/x86/utils.h @@ -1,14 +1,14 @@ #ifndef PERNIX_X86_UTILS_H #define PERNIX_X86_UTILS_H -#include +#include namespace pernix::x86::internal { - static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements) { - const u32 tail_bits = remaining_elements * bit_width; - const u32 tail_bytes = (tail_bits + 7u) / 8u; - return tail_bytes; - } +static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements) { + const u32 tail_bits = remaining_elements * bit_width; + const u32 tail_bytes = (tail_bits + 7u) / 8u; + return tail_bytes; +} } // namespace pernix::x86::internal #endif // PERNIX_X86_UTILS_H From bd12a92d6590f29ac88afd0ec03d489be1cf9b13 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Wed, 8 Jul 2026 20:36:02 +0200 Subject: [PATCH 17/43] WIP --- CMakeLists.txt | 5 +- include/pernix/dispatch/select_impl.h | 388 ++++++++++++++++-- include/pernix/fallback/common.h | 41 ++ include/pernix/fallback/scalar_compression.h | 2 + .../pernix/fallback/scalar_decompression.h | 7 +- include/pernix/fallback/stdpar_common.h | 166 ++++++++ include/pernix/fallback/stdpar_compression.h | 118 +++++- .../pernix/fallback/stdpar_decompression.h | 130 ++++-- src/CMakeLists.txt | 12 +- tests/fallback_tests.cpp | 366 +++++++++++++++++ tests/header_only_tests.cpp | 6 + 11 files changed, 1149 insertions(+), 92 deletions(-) create mode 100644 include/pernix/fallback/common.h create mode 100644 include/pernix/fallback/stdpar_common.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 899aae3..a5d4172 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,9 +13,8 @@ option(PERNIX_ENABLE_X86_AVX2 "Build x86 AVX2 backend" ON) option(PERNIX_ENABLE_X86_AVX512VBMI "Build x86 AVX512-VBMI backend" ON) option(PERNIX_ENABLE_ARM64_NEON "Build arm64 NEON backend" ON) option(PERNIX_ENABLE_ARM64_SVE2 "Build arm64 SVE2 backend" ON) -option(PERNIX_ENABLE_FALLBACK_STDPAR "Build fallback stdpar skeleton backend" OFF) +option(PERNIX_ENABLE_FALLBACK_STDPAR "Build fallback stdpar backend" ON) option(PERNIX_ENABLE_FALLBACK_SIMD "Build fallback simd skeleton backend" OFF) -option(PERNIX_ENABLE_FALLBACK_STDPAR_NVCC "Enable NVCC-oriented stdpar skeleton wiring" OFF) option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" off) @@ -73,7 +72,7 @@ else () endif () message(STATUS "Pernix version: ${VERSION}, normalized to ${NORMALIZED_VERSION}") -if (NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|Intel") +if (NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|Intel|NVHPC") message(FATAL_ERROR "Unsupported compiler: ${CMAKE_CXX_COMPILER_ID}") endif () diff --git a/include/pernix/dispatch/select_impl.h b/include/pernix/dispatch/select_impl.h index 86aae90..8cc578a 100644 --- a/include/pernix/dispatch/select_impl.h +++ b/include/pernix/dispatch/select_impl.h @@ -1146,33 +1146,193 @@ case N: return Kernel("fallback", &compress_blocks_fallback return select_fallback_compress_blocks_f64(bit_width, block_size); } - inline Kernel select_fallback_stdpar_compress_block_f32(const u8, const u32) { + #define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(N, BS) \ + case N: return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + + #define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(N, BS) \ + case N: return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + + #define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(N, BS) \ + case N: return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + + #define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(N, BS) \ + case N: return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + + #define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(24, BS); \ + default: return {"fallback_stdpar", nullptr}; \ + } + + #define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(24, BS); \ + default: return {"fallback_stdpar", nullptr}; \ + } + + #define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(24, BS); \ + default: return {"fallback_stdpar", nullptr}; \ + } + + #define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(24, BS); \ + default: return {"fallback_stdpar", nullptr}; \ + } + + inline Kernel select_fallback_stdpar_compress_block_f32(const u8 bit_width, const u32 block_size) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - return {"fallback_stdpar", &compress_block_fallback_stdpar<1, 64, f32>}; + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(1024); + default: return {"fallback_stdpar", nullptr}; + } #else return {"unsupported_implementation", nullptr}; #endif } - inline Kernel select_fallback_stdpar_compress_blocks_f32(const u8, const u32) { + inline Kernel select_fallback_stdpar_compress_blocks_f32(const u8 bit_width, const u32 block_size) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - return {"fallback_stdpar", &compress_blocks_fallback_stdpar<1, 64, f32>}; + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(1024); + default: return {"fallback_stdpar", nullptr}; + } #else return {"unsupported_implementation", nullptr}; #endif } - inline Kernel select_fallback_stdpar_compress_block_f64(const u8, const u32) { + inline Kernel select_fallback_stdpar_compress_block_f64(const u8 bit_width, const u32 block_size) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - return {"fallback_stdpar", &compress_block_fallback_stdpar<1, 64, f64>}; + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(1024); + default: return {"fallback_stdpar", nullptr}; + } #else return {"unsupported_implementation", nullptr}; #endif } - inline Kernel select_fallback_stdpar_compress_blocks_f64(const u8, const u32) { + inline Kernel select_fallback_stdpar_compress_blocks_f64(const u8 bit_width, const u32 block_size) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - return {"fallback_stdpar", &compress_blocks_fallback_stdpar<1, 64, f64>}; + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(1024); + default: return {"fallback_stdpar", nullptr}; + } #else return {"unsupported_implementation", nullptr}; #endif @@ -1437,53 +1597,209 @@ case N: \ return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); } - inline Kernel select_fallback_stdpar_decompress_block_f32(const u8, const u32, + #define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) + + #define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) + + #define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) + + #define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) + + #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(24, BS); \ + default: return {"fallback_stdpar", nullptr}; \ + } \ + break + + #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(24, BS); \ + default: return {"fallback_stdpar", nullptr}; \ + } \ + break + + #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(24, BS); \ + default: return {"fallback_stdpar", nullptr}; \ + } \ + break + + #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(24, BS); \ + default: return {"fallback_stdpar", nullptr}; \ + } \ + break + + inline Kernel select_fallback_stdpar_decompress_block_f32(const u8 bit_width, const u32 block_size, const bool sign_values) { - static_cast(sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - if (sign_values) { - return {"fallback_stdpar", &decompress_block_fallback_stdpar<1, true, 64, f32>}; + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(1024); + default: return {"fallback_stdpar", nullptr}; } - return {"fallback_stdpar", &decompress_block_fallback_stdpar<1, false, 64, f32>}; #else return {"unsupported_implementation", nullptr}; #endif } - inline Kernel select_fallback_stdpar_decompress_blocks_f32(const u8, const u32, + inline Kernel select_fallback_stdpar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, const bool sign_values) { - static_cast(sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - if (sign_values) { - return {"fallback_stdpar", &decompress_blocks_fallback_stdpar<1, true, 64, f32>}; + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(1024); + default: return {"fallback_stdpar", nullptr}; } - return {"fallback_stdpar", &decompress_blocks_fallback_stdpar<1, false, 64, f32>}; #else return {"unsupported_implementation", nullptr}; #endif } - inline Kernel select_fallback_stdpar_decompress_block_f64(const u8, const u32, + inline Kernel select_fallback_stdpar_decompress_block_f64(const u8 bit_width, const u32 block_size, const bool sign_values) { - static_cast(sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - if (sign_values) { - return {"fallback_stdpar", &decompress_block_fallback_stdpar<1, true, 64, f64>}; + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(1024); + default: return {"fallback_stdpar", nullptr}; } - return {"fallback_stdpar", &decompress_block_fallback_stdpar<1, false, 64, f64>}; #else return {"unsupported_implementation", nullptr}; #endif } - inline Kernel select_fallback_stdpar_decompress_blocks_f64(const u8, const u32, + inline Kernel select_fallback_stdpar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, const bool sign_values) { - static_cast(sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - if (sign_values) { - return {"fallback_stdpar", &decompress_blocks_fallback_stdpar<1, true, 64, f64>}; + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(1024); + default: return {"fallback_stdpar", nullptr}; } - return {"fallback_stdpar", &decompress_blocks_fallback_stdpar<1, false, 64, f64>}; #else return {"unsupported_implementation", nullptr}; #endif @@ -1741,6 +2057,14 @@ case N: return Kernel("avx2", &mm256_compress_blocks_avx2 + +namespace pernix::internal { +__always_inline f32 dequantize_epi32_ps(const i32 input, const f32 scale) { + return static_cast(input) * scale; +} + +__always_inline f64 dequantize_epi32_pd(const i64 input, const f64 scale) { + return static_cast(input) * scale; +} + +__always_inline i32 quantize_ps_epi32(const f32 input, const f32 scale) { + return static_cast(std::lroundf(input * scale)); +} + +__always_inline i64 quantize_pd_epi64(const f64 input, const f64 scale) { + return std::llround(input * scale); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 32) +__always_inline i32 sign_extend(const u32 value) { + if constexpr (BIT_WIDTH == 1) { + return static_cast(value & 1U); + } + + constexpr u32 sign_bit = 1 << (BIT_WIDTH - 1); + constexpr u32 mask = (1 << BIT_WIDTH) - 1U; + + const u32 masked = value & mask; + + return static_cast( + static_cast(masked ^ sign_bit) - static_cast(sign_bit) + ); +} +} + +#endif //PERNIX_FALLBACK_COMMON_H diff --git a/include/pernix/fallback/scalar_compression.h b/include/pernix/fallback/scalar_compression.h index 18e17cb..e478826 100644 --- a/include/pernix/fallback/scalar_compression.h +++ b/include/pernix/fallback/scalar_compression.h @@ -1,6 +1,8 @@ #ifndef PERNIX_FALLBACK_SCALAR_COMPRESSION_H #define PERNIX_FALLBACK_SCALAR_COMPRESSION_H +#include + #include #include #include diff --git a/include/pernix/fallback/scalar_decompression.h b/include/pernix/fallback/scalar_decompression.h index 50556b4..1b2e68a 100644 --- a/include/pernix/fallback/scalar_decompression.h +++ b/include/pernix/fallback/scalar_decompression.h @@ -1,6 +1,8 @@ #ifndef PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H #define PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H +#include + #include #include #include @@ -30,9 +32,8 @@ __always_inline auto sign_extend(const u32 value) -> i32 { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) -__always_inline auto unpack_epi32_fallback_inner(const u8* __restrict__ input, const u8 bit_offset, - const std::size_t elements) - -> std::vector { +__always_inline std::vector unpack_epi32_fallback_inner(const u8* __restrict__ input, const u8 bit_offset, + const std::size_t elements) { constexpr u32 bits_in_type = sizeof(T) * 8; constexpr u32 bitmask = BIT_WIDTH == bits_in_type ? std::numeric_limits::max() diff --git a/include/pernix/fallback/stdpar_common.h b/include/pernix/fallback/stdpar_common.h new file mode 100644 index 0000000..cc179d8 --- /dev/null +++ b/include/pernix/fallback/stdpar_common.h @@ -0,0 +1,166 @@ +#ifndef PERNIX_FALLBACK_STDPAR_COMMON_H +#define PERNIX_FALLBACK_STDPAR_COMMON_H + +#include + +#include +#include +#include +#include + +namespace pernix::internal { +template +struct stdpar_buffer { + std::array data{}; + usize size = 0; +}; + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__always_inline i32 sign_extend_stdpar(const u32 value) { + if constexpr (BIT_WIDTH == 1) { + return static_cast(value & 1U); + } + + constexpr u32 sign_bit = u32{1} << (BIT_WIDTH - 1); + constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1U; + const u32 masked = value & mask; + return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); +} + +template + requires(GROUP_SIZE >= 1 && GROUP_SIZE <= 8 && BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +inline constexpr usize packed_group_bytes_v = (static_cast(GROUP_SIZE) * BIT_WIDTH + 7U) / 8U; + +template + requires(std::is_floating_point_v) +__always_inline ScaleType dequantize_stdpar_value(const i32 input, const ScaleType scale) { + if constexpr (std::is_same_v) { + return static_cast(input) * scale; + } else { + return static_cast(input) * scale; + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +__always_inline u32 quantize_stdpar_value(const ScaleType input, const ScaleType scale) { + constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); + constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); + + const long double scaled = static_cast(input) * static_cast(scale); + if (std::isnan(scaled)) { + return 0; + } + if (scaled <= static_cast(min_value)) { + return static_cast(static_cast(min_value)); + } + if (scaled >= static_cast(max_value)) { + return static_cast(static_cast(max_value)); + } + + return static_cast(static_cast(std::llround(scaled))); +} + +template +class counting_iterator { +public: + using iterator_category = std::random_access_iterator_tag; + using iterator_concept = std::random_access_iterator_tag; + using value_type = T; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = T; + + constexpr explicit counting_iterator(T value) noexcept : value_(value) { + } + + constexpr T operator*() const noexcept { + return value_; + } + + constexpr T operator[](difference_type n) const noexcept { + return value_ + n; + } + + constexpr counting_iterator& operator++() noexcept { + ++value_; + return *this; + } + + constexpr counting_iterator operator++(i32) noexcept { + counting_iterator tmp = *this; + ++(*this); + return tmp; + } + + constexpr counting_iterator& operator--() noexcept { + --value_; + return *this; + } + + constexpr counting_iterator operator--(i32) noexcept { + counting_iterator tmp = *this; + --(*this); + return tmp; + } + + constexpr counting_iterator& operator+=(difference_type n) noexcept { + value_ += static_cast(n); + return *this; + } + + constexpr counting_iterator& operator-=(difference_type n) noexcept { + value_ -= static_cast(n); + return *this; + } + + friend constexpr auto operator+(counting_iterator it, difference_type n) noexcept -> counting_iterator { + it += n; + return it; + } + + friend constexpr auto operator+(difference_type n, counting_iterator it) noexcept -> counting_iterator { + it += n; + return it; + } + + friend constexpr auto operator-(counting_iterator it, difference_type n) noexcept -> counting_iterator { + it -= n; + return it; + } + + friend constexpr difference_type operator-(counting_iterator lhs, counting_iterator rhs) noexcept { + return static_cast(lhs.value_) - static_cast(rhs.value_); + } + + friend constexpr bool operator==(counting_iterator lhs, counting_iterator rhs) noexcept { + return lhs.value_ == rhs.value_; + } + + friend constexpr bool operator!=(counting_iterator lhs, counting_iterator rhs) noexcept { + return !(lhs == rhs); + } + + friend constexpr bool operator<(counting_iterator lhs, counting_iterator rhs) noexcept { + return lhs.value_ < rhs.value_; + } + + friend constexpr bool operator<=(counting_iterator lhs, counting_iterator rhs) noexcept { + return lhs.value_ <= rhs.value_; + } + + friend constexpr bool operator>(counting_iterator lhs, counting_iterator rhs) noexcept { + return lhs.value_ > rhs.value_; + } + + friend constexpr bool operator>=(counting_iterator lhs, counting_iterator rhs) noexcept { + return lhs.value_ >= rhs.value_; + } + +private: + T value_; +}; +} + +#endif // PERNIX_FALLBACK_STDPAR_COMMON_H diff --git a/include/pernix/fallback/stdpar_compression.h b/include/pernix/fallback/stdpar_compression.h index 79ff59c..5f494d9 100644 --- a/include/pernix/fallback/stdpar_compression.h +++ b/include/pernix/fallback/stdpar_compression.h @@ -1,39 +1,115 @@ #ifndef PERNIX_FALLBACK_STDPAR_COMPRESSION_H #define PERNIX_FALLBACK_STDPAR_COMPRESSION_H +#include #include +#include #include +#include #include namespace pernix { +namespace internal { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && GROUP_SIZE >= 1 && GROUP_SIZE <= 8 && std::is_floating_point_v) +__always_inline stdpar_buffer +quantize_epi32_group_fallback_stdpar(const std::span input, const ScaleType scale) { + stdpar_buffer output; + +#pragma GCC unroll GROUP_SIZE + for (usize i = 0; i < GROUP_SIZE; ++i) { + output.data[i] = quantize_stdpar_value(input[i], scale); + ++output.size; + } + + return output; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && GROUP_SIZE >= 1 && GROUP_SIZE <= 8) +__always_inline void pack_epi32_group_fallback_stdpar(const stdpar_buffer& input, + const std::span destination) { + constexpr u32 bitmask = (u32{1} << BIT_WIDTH) - 1U; + + usize idx = 0; + usize bits_in_buffer = 0; + u64 buffer = 0; + +#pragma GCC unroll GROUP_SIZE + for (usize i = 0; i < GROUP_SIZE; ++i) { + buffer |= (static_cast(input.data[i] & bitmask) << bits_in_buffer); + bits_in_buffer += BIT_WIDTH; + + while (bits_in_buffer >= 8) { + destination[idx++] = static_cast(buffer & 0xFFU); + buffer >>= 8; + bits_in_buffer -= 8; + } + } + + if (bits_in_buffer > 0) { + destination[idx] = static_cast(buffer & 0xFFU); + } +} +} + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int compress_block_fallback_stdpar(const void *input_ptr, ScaleType scale, void *output_ptr) { - [[maybe_unused]] const auto policy = std::execution::par_unseq; - [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; -#if defined(PERNIX_FALLBACK_STDPAR_USE_NVCC) - [[maybe_unused]] constexpr bool nvcc_execution_path = true; -#endif - static_cast(input_ptr); - static_cast(scale); - static_cast(output_ptr); - static_cast(elements_per_block); - return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +int compress_block_fallback_stdpar(const void* input_ptr, ScaleType scale, void* output_ptr) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + constexpr u32 full_groups = elements_per_block / 8U; + constexpr u32 remainder = elements_per_block % 8U; + + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + std::memset(output, 0, BLOCK_SIZE); + + auto first = internal::counting_iterator{0}; + std::for_each_n(std::execution::par_unseq, first, full_groups, [&](const u32 group) { + constexpr u8 group_size = 8; + constexpr usize packed_group_bytes = internal::packed_group_bytes_v; + + const auto quantized = internal::quantize_epi32_group_fallback_stdpar( + std::span(input + static_cast(group) * group_size, group_size), scale); + internal::pack_epi32_group_fallback_stdpar( + quantized, std::span(output + static_cast(group) * BIT_WIDTH, packed_group_bytes)); + }); + + if constexpr (remainder != 0) { + constexpr u8 tail_group_size = static_cast(remainder); + constexpr usize tail_input_offset = full_groups * 8U; + constexpr usize tail_output_offset = full_groups * BIT_WIDTH; + constexpr usize tail_bytes = internal::packed_group_bytes_v; + + const auto quantized = internal::quantize_epi32_group_fallback_stdpar( + std::span(input + tail_input_offset, tail_group_size), scale); + internal::pack_epi32_group_fallback_stdpar( + quantized, std::span(output + tail_output_offset, tail_bytes)); + } + + return PERNIX_STATUS_OK; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int compress_blocks_fallback_stdpar(const void *input_ptr, ScaleType scale, void *output_ptr, u32 blocks) { - [[maybe_unused]] const auto policy = std::execution::par_unseq; -#if defined(PERNIX_FALLBACK_STDPAR_USE_NVCC) - [[maybe_unused]] constexpr bool nvcc_execution_path = true; -#endif - static_cast(input_ptr); - static_cast(scale); - static_cast(output_ptr); - static_cast(blocks); - return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +int compress_blocks_fallback_stdpar(const void* input_ptr, ScaleType scale, void* output_ptr, u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + auto first = internal::counting_iterator{0}; + std::for_each_n(std::execution::par_unseq, first, blocks, [&](const u32 block) { + const auto* block_input = input + (static_cast(block) * elements_per_block); + auto* block_output = output + (static_cast(block) * BLOCK_SIZE); + static_cast(compress_block_fallback_stdpar(block_input, + scale, + block_output)); + }); + + return PERNIX_STATUS_OK; } } diff --git a/include/pernix/fallback/stdpar_decompression.h b/include/pernix/fallback/stdpar_decompression.h index 720ded9..ffbae89 100644 --- a/include/pernix/fallback/stdpar_decompression.h +++ b/include/pernix/fallback/stdpar_decompression.h @@ -1,43 +1,113 @@ #ifndef PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H #define PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H -#include +#include #include +#include #include +#include "scalar_decompression.h" + namespace pernix { -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int decompress_block_fallback_stdpar(const void *input_ptr, ScaleType scale, void *output_ptr) { - [[maybe_unused]] const auto policy = std::execution::par_unseq; - [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; - [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; -#if defined(PERNIX_FALLBACK_STDPAR_USE_NVCC) - [[maybe_unused]] constexpr bool nvcc_execution_path = true; -#endif - static_cast(input_ptr); - static_cast(scale); - static_cast(output_ptr); - static_cast(sign_values); - static_cast(elements_per_block); - return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +namespace internal { +// Eight packed values consume exactly BIT_WIDTH bytes. +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && GROUP_SIZE >= 1 && GROUP_SIZE <= 8) +__always_inline stdpar_buffer unpack_epi32_group_fallback_stdpar(const std::span input) { + constexpr auto bit_offset = 0; // TODO: for now + constexpr u32 bitmask = (u32{1} << BIT_WIDTH) - 1U; + + stdpar_buffer output; + + // Unpack one fixed-size byte group using the scalar bit-buffer logic. + + usize idx = 0; + u8 bits_in_buffer = 8 - bit_offset; + u64 buffer = static_cast(input[idx++]) >> bit_offset; + +#pragma GCC unroll GROUP_SIZE + for (usize i = 0; i < GROUP_SIZE; i++) { + while (BIT_WIDTH > bits_in_buffer) { + const auto next_value = static_cast(input[idx++]) << bits_in_buffer; + buffer |= next_value; + bits_in_buffer += 8; + } + + const u32 raw_value = static_cast(buffer & bitmask); + if constexpr (SIGN_VALUES) { + output.data[i] = sign_extend_stdpar(raw_value); + } else { + output.data[i] = static_cast(raw_value); + } + ++output.size; + + buffer >>= BIT_WIDTH; + bits_in_buffer -= BIT_WIDTH; + } + return output; +} +} + +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int decompress_block_fallback_stdpar(const void* __restrict__ input_ptr, const ScaleType scale, + void* __restrict__ output_ptr) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 full_groups = elements_per_block / 8; + constexpr u32 remainder = elements_per_block % 8; + + const auto input_span = std::span(static_cast(input_ptr), BLOCK_SIZE); + auto output_span = std::span(static_cast(output_ptr), elements_per_block); + + auto first = internal::counting_iterator{0}; + std::for_each_n(std::execution::par_unseq, first, full_groups, [&](const u32 group) { + const auto input_subspan = input_span.subspan(static_cast(group) * BIT_WIDTH, BIT_WIDTH); + const auto unpacked = internal::unpack_epi32_group_fallback_stdpar(input_subspan); + + for (u32 i = 0; i < unpacked.size; ++i) { + output_span[static_cast(group) * 8U + i] = internal::dequantize_stdpar_value(unpacked.data[i], scale); + } + }); + + if constexpr (remainder != 0) { + constexpr usize tail_input_offset = full_groups * BIT_WIDTH; + constexpr usize tail_output_offset = full_groups * 8U; + constexpr usize tail_bytes = internal::packed_group_bytes_v(remainder), BIT_WIDTH>; + + const auto unpacked = internal::unpack_epi32_group_fallback_stdpar< + BIT_WIDTH, + SIGN_VALUES, + static_cast(remainder)>( + input_span.subspan(tail_input_offset, tail_bytes)); + + for (u32 i = 0; i < remainder; ++i) { + output_span[tail_output_offset + i] = + internal::dequantize_stdpar_value(unpacked.data[i], scale); + } + } + + return 0; } -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int decompress_blocks_fallback_stdpar(const void *input_ptr, ScaleType scale, void *output_ptr, u32 blocks) { - [[maybe_unused]] const auto policy = std::execution::par_unseq; - [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; -#if defined(PERNIX_FALLBACK_STDPAR_USE_NVCC) - [[maybe_unused]] constexpr bool nvcc_execution_path = true; -#endif - static_cast(input_ptr); - static_cast(scale); - static_cast(output_ptr); - static_cast(blocks); - static_cast(sign_values); - return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int decompress_blocks_fallback_stdpar(const void* __restrict__ input_ptr, const ScaleType scale, + void* __restrict__ output_ptr, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + auto first = internal::counting_iterator{0}; + std::for_each_n(std::execution::par_unseq, first, blocks, [&](const u32 block) { + const auto* block_input = input + (static_cast(block) * BLOCK_SIZE); + auto* block_output = output + (static_cast(block) * elements_per_block); + static_cast(decompress_block_fallback_stdpar( + block_input, scale, block_output)); + }); + + return 0; } } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 41e21c6..7ada82a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,8 +6,8 @@ add_library(pernix_header_only INTERFACE) add_library(pernix::pernix_header_only ALIAS pernix_header_only) target_sources(pernix PRIVATE pernix.cpp) -target_compile_features(pernix PUBLIC cxx_std_20) -target_compile_features(pernix_header_only INTERFACE cxx_std_20) +target_compile_features(pernix PUBLIC cxx_std_23) +target_compile_features(pernix_header_only INTERFACE cxx_std_23) target_compile_options(pernix PRIVATE ${PERNIX_PRIVATE_COMPILE_OPTIONS}) set_target_properties(pernix PROPERTIES @@ -40,8 +40,11 @@ if (PERNIX_USE_SIMDE) endif () if (PERNIX_ENABLE_FALLBACK_STDPAR) + find_package(TBB REQUIRED) target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_STDPAR=1) target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_STDPAR=1) + target_link_libraries(pernix PUBLIC TBB::tbb) + target_link_libraries(pernix_header_only INTERFACE TBB::tbb) endif () if (PERNIX_ENABLE_FALLBACK_SIMD) @@ -49,11 +52,6 @@ if (PERNIX_ENABLE_FALLBACK_SIMD) target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_SIMD=1) endif () -if (PERNIX_ENABLE_FALLBACK_STDPAR_NVCC) - target_compile_definitions(pernix PUBLIC PERNIX_FALLBACK_STDPAR_USE_NVCC=1) - target_compile_definitions(pernix_header_only INTERFACE PERNIX_FALLBACK_STDPAR_USE_NVCC=1) -endif () - if (PERNIX_ENABLE_X86_BMI2 AND PERNIX_TARGET_IS_X86) target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_BMI2=1) endif () diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp index ef57986..75f91ac 100644 --- a/tests/fallback_tests.cpp +++ b/tests/fallback_tests.cpp @@ -17,6 +17,47 @@ #endif #endif +namespace { +template +void expect_backend_compressed_block_matches_reference(FixtureT& fixture, const pernix_backend backend, + CompressFn compress_fn) { + using ScaleType = std::remove_cvref_t; + std::vector > compressed(fixture.testSet.numberOfBlocks); + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { + compressed[b].resize(FixtureT::BlockSize); + const auto status = compress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getDecompressedData()[b].data(), + ScaleType{1} / fixture.testSet.getScales()[b], + compressed[b].data()); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { + expectCompressedBlockEqualsReference(fixture, compressed[b], b); + } +} + +template +void expect_backend_decompressed_block_matches_source(FixtureT& fixture, const pernix_backend backend, + DecompressFn decompress_fn) { + using ValueType = std::remove_cvref_t; + std::vector > decompressed(fixture.testSet.numberOfBlocks); + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { + decompressed[b].resize(fixture.testSet.elementsPerBlock); + const auto status = decompress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, + fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed[b].data(), true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { + expectDecompressedBlockNearSource(fixture, decompressed[b], b); + } +} +} + // --------------------------------------------------------------------------- // Fallback compress: verify byte-exact match against the reference // --------------------------------------------------------------------------- @@ -97,6 +138,28 @@ TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { } } +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TYPED_TEST(CompressionTest, FallbackStdparCompressBlock) { + expect_backend_compressed_block_matches_reference(*this, PERNIX_BACKEND_FALLBACK_STDPAR, + &pernix_compress_block_f32); +} + +TYPED_TEST(CompressionTest64, FallbackStdparCompressBlock) { + expect_backend_compressed_block_matches_reference(*this, PERNIX_BACKEND_FALLBACK_STDPAR, + &pernix_compress_block_f64); +} + +TYPED_TEST(DecompressionTest, FallbackStdparDecompressBlock) { + expect_backend_decompressed_block_matches_source(*this, PERNIX_BACKEND_FALLBACK_STDPAR, + &pernix_decompress_block_f32); +} + +TYPED_TEST(DecompressionTest64, FallbackStdparDecompressBlock) { + expect_backend_decompressed_block_matches_source(*this, PERNIX_BACKEND_FALLBACK_STDPAR, + &pernix_decompress_block_f64); +} +#endif + // --------------------------------------------------------------------------- // Multi-block roundtrip via compress_blocks / decompress_blocks (fallback) // --------------------------------------------------------------------------- @@ -177,6 +240,76 @@ TYPED_TEST(CompressionTest64, FallbackCompressBlocksRoundtrip) { } } +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TYPED_TEST(CompressionTest, FallbackStdparCompressBlocksRoundtrip) { + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; + const u32 total = nb * epb; + + std::vector flat(total); + for (u32 b = 0; b < nb; ++b) { + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); + } + + float max_abs = 0.0f; + for (u32 i = 0; i < total; ++i) { + max_abs = std::max(max_abs, std::abs(flat[i])); + } + const float q = static_cast(decltype(this->testSet)::quantization_levels); + const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); + const float scale_inv = 1.0f / scale; + + std::vector compressed(nb * TestFixture::BlockSize); + auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, + TestFixture::BlockSize, flat.data(), scale_inv, compressed.data(), nb); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + std::vector restored(total); + status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, + TestFixture::BlockSize, compressed.data(), scale, restored.data(), nb, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + const float tol = (std::abs(scale) * 0.5f) + (std::numeric_limits::epsilon() * 16.0f); + for (u32 i = 0; i < total; ++i) { + EXPECT_NEAR(restored[i], flat[i], tol); + } +} + +TYPED_TEST(CompressionTest64, FallbackStdparCompressBlocksRoundtrip) { + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; + const u32 total = nb * epb; + + std::vector flat(total); + for (u32 b = 0; b < nb; ++b) { + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); + } + + double max_abs = 0.0; + for (u32 i = 0; i < total; ++i) { + max_abs = std::max(max_abs, std::abs(flat[i])); + } + const double q = static_cast(decltype(this->testSet)::quantization_levels); + const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); + const double scale_inv = 1.0 / scale; + + std::vector compressed(nb * TestFixture::BlockSize); + auto status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, + TestFixture::BlockSize, flat.data(), scale_inv, compressed.data(), nb); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + std::vector restored(total); + status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, + TestFixture::BlockSize, compressed.data(), scale, restored.data(), nb, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + const double tol = (std::abs(scale) * 0.5) + (std::numeric_limits::epsilon() * 16.0); + for (u32 i = 0; i < total; ++i) { + EXPECT_NEAR(restored[i], flat[i], tol); + } +} +#endif + // --------------------------------------------------------------------------- // blocks API with a single block should match the block API exactly // --------------------------------------------------------------------------- @@ -226,6 +359,50 @@ TYPED_TEST(DecompressionTest, SingleBlockDecompressBlocksMatchesBlock) { } } +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TYPED_TEST(CompressionTest, SingleBlockStdparCompressBlocksMatchesBlock) { + const auto& src = this->testSet.getDecompressedData()[0]; + const float scale_inv = 1.0f / this->testSet.getScales()[0]; + + std::vector block_out(TestFixture::BlockSize); + std::vector blocks_out(TestFixture::BlockSize); + + auto s1 = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, + TestFixture::BlockSize, src.data(), scale_inv, block_out.data()); + ASSERT_EQ(s1, PERNIX_STATUS_OK); + + auto s2 = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, + TestFixture::BlockSize, src.data(), scale_inv, blocks_out.data(), 1); + ASSERT_EQ(s2, PERNIX_STATUS_OK); + + for (u32 i = 0; i < TestFixture::BlockSize; ++i) { + EXPECT_EQ(block_out[i], blocks_out[i]) << "byte " << i; + } +} + +TYPED_TEST(DecompressionTest, SingleBlockStdparDecompressBlocksMatchesBlock) { + const auto& compressed = this->testSet.getCompressedData()[0]; + const float scale = this->testSet.getScales()[0]; + const u32 epb = this->testSet.elementsPerBlock; + + std::vector block_out(epb); + std::vector blocks_out(epb); + + auto s1 = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, + TestFixture::BlockSize, compressed.data(), scale, block_out.data(), true); + ASSERT_EQ(s1, PERNIX_STATUS_OK); + + auto s2 = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, + TestFixture::BlockSize, compressed.data(), scale, blocks_out.data(), 1, + true); + ASSERT_EQ(s2, PERNIX_STATUS_OK); + + for (u32 i = 0; i < epb; ++i) { + EXPECT_FLOAT_EQ(block_out[i], blocks_out[i]) << "element " << i; + } +} +#endif + // --------------------------------------------------------------------------- // Edge-case behavioural tests (fallback, block_size=64) // --------------------------------------------------------------------------- @@ -280,6 +457,190 @@ TEST(FallbackEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { EXPECT_EQ(restored[2], 0.0f); } +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TEST(FallbackStdparEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { + constexpr u32 BS = 64; + const std::array input{0x08}; + + std::array output{}; + const auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, 4, BS, input.data(), 1.0f, + output.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + EXPECT_EQ(output[0], -8.0f); +} + +TEST(FallbackStdparEdgeTest, ClearsUnusedPaddingBytes) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + std::array output{}; + output.fill(0xA5); + + const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, + output.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + EXPECT_EQ(output[BS - 1], 0); +} + +TEST(FallbackStdparEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { + constexpr u32 BS = 64; + constexpr u32 BW = 4; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + input[0] = std::numeric_limits::infinity(); + input[1] = -std::numeric_limits::infinity(); + input[2] = std::numeric_limits::quiet_NaN(); + + std::array compressed{}; + std::array restored{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, + compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, + restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(restored[0], 7.0f); + EXPECT_EQ(restored[1], -8.0f); + EXPECT_EQ(restored[2], 0.0f); +} + +TEST(FallbackStdparEdgeTest, TailWidthCompressMatchesScalarF32) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < EPB; ++i) { + input[i] = static_cast(static_cast(i) - 10); + } + + std::array scalar_block{}; + std::array stdpar_block{}; + std::array stdpar_blocks{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, scalar_block.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, stdpar_block.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, + stdpar_blocks.data(), 1); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + for (u32 i = 0; i < BS; ++i) { + EXPECT_EQ(stdpar_block[i], scalar_block[i]) << "block byte " << i; + EXPECT_EQ(stdpar_blocks[i], scalar_block[i]) << "blocks byte " << i; + } +} + +TEST(FallbackStdparEdgeTest, TailWidthCompressMatchesScalarF64) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < EPB; ++i) { + input[i] = static_cast((static_cast(i) - 10) * 2); + } + + std::array scalar_block{}; + std::array stdpar_block{}; + std::array stdpar_blocks{}; + + auto st = pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0, scalar_block.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0, stdpar_block.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0, + stdpar_blocks.data(), 1); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + for (u32 i = 0; i < BS; ++i) { + EXPECT_EQ(stdpar_block[i], scalar_block[i]) << "block byte " << i; + EXPECT_EQ(stdpar_blocks[i], scalar_block[i]) << "blocks byte " << i; + } +} + +TEST(FallbackStdparEdgeTest, TailWidthDecompressMatchesScalarF32) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < EPB; ++i) { + input[i] = static_cast(static_cast(i) - 10); + } + + std::array compressed{}; + std::array scalar_block{}; + std::array stdpar_block{}; + std::array stdpar_blocks{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), 1.0f, scalar_block.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, + stdpar_block.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, + stdpar_blocks.data(), 1, true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + for (u32 i = 0; i < EPB; ++i) { + EXPECT_FLOAT_EQ(stdpar_block[i], scalar_block[i]) << "block element " << i; + EXPECT_FLOAT_EQ(stdpar_blocks[i], scalar_block[i]) << "blocks element " << i; + } +} + +TEST(FallbackStdparEdgeTest, TailWidthDecompressMatchesScalarF64) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < EPB; ++i) { + input[i] = static_cast((static_cast(i) - 10) * 2); + } + + std::array compressed{}; + std::array scalar_block{}; + std::array stdpar_block{}; + std::array stdpar_blocks{}; + + auto st = pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0, compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), 1.0, scalar_block.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0, + stdpar_block.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0, + stdpar_blocks.data(), 1, true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + for (u32 i = 0; i < EPB; ++i) { + EXPECT_DOUBLE_EQ(stdpar_block[i], scalar_block[i]) << "block element " << i; + EXPECT_DOUBLE_EQ(stdpar_blocks[i], scalar_block[i]) << "blocks element " << i; + } +} +#endif + // --------------------------------------------------------------------------- // Error-code contract tests // --------------------------------------------------------------------------- @@ -336,8 +697,13 @@ TEST(ErrorCodeTest, FallbackStdparReturnsUnsupportedImplementationByDefault) { f32 src[(BS * 8U) / BW] = {}; u8 dst[BS] = {}; +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_OK); +#else const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, src, 1.0f, dst); EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +#endif } TEST(ErrorCodeTest, FallbackSimdReturnsUnsupportedImplementationByDefault) { diff --git a/tests/header_only_tests.cpp b/tests/header_only_tests.cpp index 211667c..77e900d 100644 --- a/tests/header_only_tests.cpp +++ b/tests/header_only_tests.cpp @@ -50,9 +50,15 @@ TEST(HeaderOnlyPernix, FallbackStdparReturnsUnsupportedImplementation) { auto input = make_input(); std::vector compressed(kBlockSize); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, kBitWidth, kBlockSize, + std::span(input), 1.0f, std::span(compressed)), + PERNIX_STATUS_OK); +#else EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, kBitWidth, kBlockSize, std::span(input), 1.0f, std::span(compressed)), PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +#endif } TEST(HeaderOnlyPernix, FallbackSimdReturnsUnsupportedImplementation) { From f8ca1a06caf4ec579d10dc7bd8ca3edb79fe506b Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Wed, 8 Jul 2026 21:12:35 +0200 Subject: [PATCH 18/43] feat: enhance Fortran bindings with new compression and decompression functions, add scale helpers, and improve CMake configuration --- CMakeLists.txt | 9 +- README.md | 201 ++++++++++++++++--------- bindings/README.md | 12 +- bindings/fortran/CMakeLists.txt | 16 +- bindings/fortran/main.f90 | 60 ++++---- bindings/fortran/src/compression.f90 | 116 +++++--------- bindings/fortran/src/decompression.f90 | 117 +++++--------- include/pernix/compat.h | 12 +- include/pernix/detail/api.hpp | 59 +++++++- include/pernix/pernix.h | 20 +++ include/pernix/pernix.hpp | 103 +++++++++++++ src/pernix.cpp | 32 ++++ tests/CMakeLists.txt | 5 + tests/fallback_tests.cpp | 68 +++++++++ tests/header_only_tests.cpp | 86 +++++++++++ 15 files changed, 633 insertions(+), 283 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5d4172..b9aea32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ option(PERNIX_ENABLE_LTO "Enable Link Time Optimization (LTO)" off) option(PERNIX_ENABLE_INSTALL "Enable installation of pernix" on) option(PERNIX_ENABLE_DOXYGEN "Build documentation with Doxygen." off) option(PERNIX_INSTALL_DOCS "Install documentation with pernix" off) +option(PERNIX_ENABLE_EXAMPLES "Build pernix example programs" off) option(PERNIX_ENABLE_X86_BMI2 "Build x86 BMI2 backend" ON) option(PERNIX_ENABLE_X86_AVX2 "Build x86 AVX2 backend" ON) @@ -144,8 +145,12 @@ add_subdirectory(src) if (PERNIX_ENABLE_FORTRAN_BINDINGS) message(STATUS "Building Fortran bindings for pernix") - include(CMakeAddFortranSubdirectory) - cmake_add_fortran_subdirectory(bindings/fortran NO_EXTERNAL_INSTALL) + enable_language(Fortran) + add_subdirectory(bindings/fortran) +endif () + +if (PERNIX_ENABLE_EXAMPLES) + add_subdirectory(examples) endif () if (PERNIX_ENABLE_TESTS) diff --git a/README.md b/README.md index 7c018d9..6e6ed48 100644 --- a/README.md +++ b/README.md @@ -1,125 +1,186 @@ # PERNIX: Floating-Point Number De/Compression on CPUs -PERNIX is a high-throughput floating-point compression library for CPU-based scientific workloads. It quantizes floating-point values to a configurable bit width and packs them into fixed-size blocks, reducing memory and communication bandwidth while keeping decompression fast. + +PERNIX is a high-throughput floating-point compression library for CPU-based scientific workloads. It quantizes +floating-point values to a configurable bit width and packs them into fixed-size compressed blocks, reducing memory and +communication bandwidth while keeping decompression fast. The library provides: -* C++ template API (`pernix::compress_block`, `pernix::decompress_block`) -* C ABI wrappers (`compress_block`, `decompress_block`, and `_f64` variants) -* Fortran bindings in `bindings/fortran` -* SIMD-optimized backends (AVX2, AVX-512 VBMI, BMI2) with fallback implementations +* C++ API functions such as `pernix::compress_block` and `pernix::decompress_block` +* C ABI wrappers named `pernix_compress_block_f32`, `pernix_decompress_block_f32`, and matching `_f64` variants +* Optional Fortran bindings in `bindings/fortran` +* SIMD-optimized backends where available, with a portable fallback backend + +## Quantization Convention + +For a block of floating-point numbers `x_i` and a bit width `N`, PERNIX uses the block maximum magnitude: + +```text +bmax = max(abs(x_i)) +scale = bmax / (2^(N - 1) - 1) +``` + +The current compression API expects the inverse scale, because compression computes: + +```text +quantized = round(input * scale_inverse) +``` + +Decompression expects the forward scale, because decompression computes: + +```text +output = quantized * scale +``` + +In practice, compute `scale` from `bmax`, pass `1 / scale` to `compress_block`, and pass `scale` to +`decompress_block`. The helper functions `pernix_scale_f32`, `pernix_scale_f64`, and +`pernix::scale_from_bmax` compute this scale and return a small positive scale for `bmax == 0`. + +Scale arguments passed to compression and decompression must be finite and greater than zero. Passing zero, negative, +NaN, or infinity returns `PERNIX_STATUS_INVALID_ARGUMENT`. + +## Block Format and Limits + +PERNIX uses 64-byte (512-bit) compressed blocks by default. For bit width `N`, one 64-byte block stores +`(64 * 8) / N` values. The fallback implementation currently supports bit widths `1..24`. Public APIs also accept an +explicit `block_size`; supported values are `64`, `128`, `256`, `512`, and `1024`, but the documented default and normal +format is a fixed 64-byte block. + +Packed values are signed by default during decompression. A `sign_values` argument is available in the C ABI and in the +C++ API to request unsigned interpretation. + +The public helper APIs expose the fixed-format constants: + +* `pernix_min_bit_width()` / `pernix::min_bit_width()` return `1` +* `pernix_max_bit_width()` / `pernix::max_bit_width()` return `24` +* `pernix_compressed_block_size()` / `pernix::compressed_block_size()` return `64` +* `pernix_elements_per_block(bit_width)` / `pernix::elements_per_block(bit_width)` return `(64 * 8) / bit_width`, or + `0` for an invalid bit width + +Input values should be finite for portable behavior. The scalar fallback clamps non-finite or out-of-range values before +narrowing, but that behavior is not a cross-backend contract. + +## Status Codes -Compression of floating-point numbers $x_i$ to $N$-bit quantized numbers with scale $\varepsilon$ (M. Guidon, F. Schiffmann, J. Hutter and J. VandeVondele, The Journal of Chemical Physics, 2008, 128, 214104): +The C ABI returns `pernix_status`: -$$b_{\max} = \max\left(\lvert x_i \rvert\right) \quad \quad \varepsilon = \frac{b_{\max}}{2^{N-1}-1}$$ +* `PERNIX_STATUS_OK`: success +* `PERNIX_STATUS_INVALID_ARGUMENT`: null pointer or invalid parameter +* `PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH`: unsupported bit width +* `PERNIX_STATUS_UNSUPPORTED_BACKEND`: unknown backend value +* `PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE`: unsupported block size +* `PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION`: backend was requested but is not available for this build or CPU -$$x_{i,N} = \mathrm{ANINT}\left(x_i\cdot\varepsilon^{-1}\right)$$ +## f32 and f64 APIs -PERNIX is block-based and uses 64-byte (512-bit) compressed blocks by default. For a bit width `N`, each block stores `(64 * 8) / N` values. +Use the `_f32` functions for `float` input/output and `_f64` functions for `double` input/output. Both variants pack to +the same integer bit stream for the selected bit width. The scale type matches the floating-point type: +`float` for `_f32`, `double` for `_f64`. ## Compiling -1. clone repository `git clone https://github.com/pc2/pernix` -2. build with CMake: - * `cmake -E make_directory "build"` - * `cmake -E chdir "build" cmake -DCMAKE_BUILD_TYPE=Release -DPERNIX_ENABLE_TESTS=off ../` - * `cmake --build "build" --config Release` -3. `libpernix.so` will be in `build/src` -To enable Fortran bindings, configure with `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON`. +```bash +cmake -E make_directory build +cmake -E chdir build cmake -DCMAKE_BUILD_TYPE=Release -DPERNIX_ENABLE_TESTS=off ../ +cmake --build build --config Release +``` + +The shared library is written under `build/src`. + +Optional build flags: + +* `-DPERNIX_ENABLE_TESTS=ON` builds the test suite. +* `-DPERNIX_ENABLE_EXAMPLES=ON` builds the C and C++ examples. +* `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON` builds the Fortran bindings and Fortran round-trip program. +* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF` disables the stdpar fallback backend and its TBB dependency. ## Usage Examples -### C++ API example (single block) +### C++ API ```cpp +#include + #include #include #include -#include -#include +#include int main() { - constexpr uint8_t BIT_WIDTH = 16; - constexpr uint32_t BLOCK_SIZE = 64; - constexpr size_t ELEMENTS = (BLOCK_SIZE * 8) / BIT_WIDTH; // 32 values for 16-bit + constexpr u8 bit_width = 16; + constexpr u32 block_size = 64; + constexpr std::size_t elements = (block_size * 8U) / bit_width; - std::array input{}; - for (size_t i = 0; i < ELEMENTS; ++i) { - input[i] = std::sin(static_cast(i)); + std::array input{}; + for (std::size_t i = 0; i < input.size(); ++i) { + input[i] = std::sin(static_cast(i) * 0.25f); } float bmax = 0.0f; - for (float x : input) { - bmax = std::max(bmax, std::abs(x)); + for (float value : input) { + bmax = std::max(bmax, std::abs(value)); + } + float scale = 0.0f; + if (pernix::scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK) { + return 1; } - const float scale = bmax / ((1u << (BIT_WIDTH - 1)) - 1u); - std::array compressed{}; - std::array restored{}; + std::array compressed{}; + std::array restored{}; - if (pernix::compress_block(input.data(), scale, compressed.data()) != 0) { + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, 1.0f / scale, + compressed) != PERNIX_STATUS_OK) { return 1; } - if (pernix::decompress_block(compressed.data(), scale, restored.data()) != 0) { + if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, scale, + restored) != PERNIX_STATUS_OK) { return 1; } return 0; } ``` -### C ABI example (single block) +### C ABI ```c -#include -#include #include int main(void) { - const uint8_t bit_width = 16; - float input[32]; - uint8_t compressed[64]; - float restored[32]; - float scale = 1.0f; - - for (int i = 0; i < 32; ++i) { - input[i] = sinf((float)i); + enum { bit_width = 16, block_size = 64, elements = (block_size * 8) / bit_width }; + float input[elements]; + float restored[elements]; + u8 compressed[block_size]; + float bmax = 0.0f; + + for (int i = 0; i < elements; ++i) { + input[i] = ((float)i - 16.0f) * 0.125f; + const float magnitude = input[i] < 0.0f ? -input[i] : input[i]; + bmax = bmax < magnitude ? magnitude : bmax; } - if (compress_block(bit_width, input, scale, compressed) != 0) { + float scale = 0.0f; + if (pernix_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK) { return 1; } - if (decompress_block(bit_width, compressed, scale, restored) != 0) { + + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 1.0f / scale, + compressed) != PERNIX_STATUS_OK) { + return 1; + } + if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, + true) != PERNIX_STATUS_OK) { return 1; } return 0; } ``` -### Fortran example (using bindings) - -```fortran -program pernix_example - use iso_c_binding, only : c_int8_t, c_float, c_loc - use pernix_compression - use pernix_decompression - implicit none - - integer(c_int8_t), parameter :: bit_width = 16_c_int8_t - real(c_float), parameter :: scale = 1.5_c_float - real(c_float), target :: input_data(32), output_data(32) - integer(c_int8_t), target :: compressed_data(64) - integer :: i +### Fortran - do i = 1, size(input_data) - input_data(i) = real(i, c_float) - end do - - call compress_block(bit_width, c_loc(input_data), scale, c_loc(compressed_data)) - call decompress_block(bit_width, c_loc(compressed_data), scale, c_loc(output_data)) -end program pernix_example -``` - -For a complete Fortran binding setup, see `bindings/README.md` and `bindings/fortran/main.f90`. +Fortran bindings call the same C ABI symbols. Enable them with `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON`. A minimal +round-trip program is provided in `bindings/fortran/main.f90`, and a standalone example is in +`examples/fortran/roundtrip.f90`. ## Benchmarking -A benchmark framework for PERNIX can be found at https://github.com/pc2/pernix-benchmark. +A benchmark framework for PERNIX can be found at https://github.com/pc2/pernix-benchmark. diff --git a/bindings/README.md b/bindings/README.md index f3ae472..d5c0273 100644 --- a/bindings/README.md +++ b/bindings/README.md @@ -1,14 +1,14 @@ ## Fortran bindings -Fortran bindings for the library are provided in the `bindings/fortran` directory. These bindings allow Fortran programs -to interface with the core functionality of the library. +Fortran bindings for the library are provided in the `bindings/fortran` directory. They expose the public C ABI symbols +such as `pernix_compress_block_f32`, `pernix_decompress_block_f32`, and the matching f64 variants. -### Building the Fortran Bindings +### Building the Fortran bindings -To build the Fortran bindings, you need to have a Fortran compiler installed on your system. The build process is -integrated into the main build system of the library. You can enable the Fortran bindings by passing the appropriate -flag to the build configuration command. For example, if you are using CMake, you can enable the Fortran bindings with: +To build the Fortran bindings, you need a Fortran compiler. Enable them from the main CMake build: ```bash cmake -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON .. ``` + +The `pernix_bindings_fortran_test` target performs one f32 64-byte block round trip through the Fortran modules. diff --git a/bindings/fortran/CMakeLists.txt b/bindings/fortran/CMakeLists.txt index ce9995b..c745cf8 100644 --- a/bindings/fortran/CMakeLists.txt +++ b/bindings/fortran/CMakeLists.txt @@ -1,19 +1,23 @@ -enable_language(Fortran) - file(GLOB_RECURSE SOURCE_FILES src/*.f90 ) add_library(pernix_bindings_fortran SHARED ${SOURCE_FILES}) -target_sources(pernix_bindings_fortran PRIVATE ${SOURCE_FILES}) +set_target_properties(pernix_bindings_fortran PROPERTIES + Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/mod" +) +target_include_directories(pernix_bindings_fortran PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/mod") target_link_libraries(pernix_bindings_fortran PUBLIC pernix) add_executable(pernix_bindings_fortran_test main.f90) target_link_libraries(pernix_bindings_fortran_test PRIVATE pernix_bindings_fortran pernix) -link_directories($ENV{LD_LIBRARY_PATH}) find_package(OpenMP) -if (OpenMP_CXX_FOUND) - target_link_libraries(pernix_bindings_fortran_test PUBLIC OpenMP::OpenMP_CXX) +if (OpenMP_Fortran_FOUND) + target_link_libraries(pernix_bindings_fortran_test PUBLIC OpenMP::OpenMP_Fortran) +endif () + +if (BUILD_TESTING) + add_test(NAME pernix_fortran_roundtrip COMMAND pernix_bindings_fortran_test) endif () diff --git a/bindings/fortran/main.f90 b/bindings/fortran/main.f90 index d1cac69..2f499e8 100644 --- a/bindings/fortran/main.f90 +++ b/bindings/fortran/main.f90 @@ -1,42 +1,36 @@ -! Hello World Fortran program -program main - use iso_c_binding, only : c_int8_t, c_int32_t, c_float, c_loc, c_ptr - use iso_fortran_env, only : real64, int64 - use pernix_decompression +program pernix_fortran_roundtrip + use iso_c_binding, only : c_bool, c_float, c_int, c_int8_t, c_int32_t, c_loc use pernix_compression - use omp_lib + use pernix_decompression implicit none - integer(c_int8_t) :: bit_width = 16_c_int8_t + integer(c_int8_t), parameter :: bit_width = 16_c_int8_t + integer(c_int32_t), parameter :: block_size = 64_c_int32_t + integer, parameter :: elements = (64 * 8) / 16 + real(c_float), target :: input(elements) + real(c_float), target :: restored(elements) + integer(c_int8_t), target :: compressed(block_size) + real(c_float) :: bmax real(c_float) :: scale - integer(c_int8_t), target :: input_data(512) - real(c_float), target :: output_data(512) - integer :: i, j, gb, iter - real(real64) :: t0, t1, diff + integer(c_int) :: status + integer :: i - ! Initialize example data - scale = 1.5_c_float - output_data = 0.0_c_float + do i = 1, elements + input(i) = real(i - 17, c_float) * 0.125_c_float + end do - iter = 50000000_int64 - t0 = omp_get_wtime() - do i = 1_int64, iter - do j = 1, size(input_data) - input_data(j) = mod(i + j, 256_int64) - end do + bmax = maxval(abs(input)) + scale = bmax / real((2 ** (int(bit_width) - 1)) - 1, c_float) - ! call the C-binding Fortran subroutine using C pointers - call decompress_block(bit_width, c_loc(input_data), scale, c_loc(output_data)) - end do - t1 = omp_get_wtime() - diff = t1 - t0 - gb = iter * (512_int64 + 1024_int64) / (1024_int64 * 1024_int64 * 1024_int64) + status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(input), & + 1.0_c_float / scale, c_loc(compressed)) + if (status /= PERNIX_STATUS_OK) stop 1 - print *, "Time taken for ", iter, " decompressions: ", diff, " seconds" - print *, "Throughput: ", gb / diff, " GB/s" + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(compressed), & + scale, c_loc(restored), .true._c_bool) + if (status /= PERNIX_STATUS_OK) stop 2 - ! Print the output data - ! do i = 1, size(output_data) - ! print *, "output_data(", i, ") = ", output_data(i) - ! end do -end program main \ No newline at end of file + do i = 1, elements + if (abs(restored(i) - input(i)) > scale) stop 3 + end do +end program pernix_fortran_roundtrip diff --git a/bindings/fortran/src/compression.f90 b/bindings/fortran/src/compression.f90 index 8261364..d294ee1 100644 --- a/bindings/fortran/src/compression.f90 +++ b/bindings/fortran/src/compression.f90 @@ -1,104 +1,60 @@ module pernix_compression - use iso_c_binding, only : c_int8_t, c_int32_t, c_float, c_ptr + use iso_c_binding, only : c_bool, c_double, c_float, c_int, c_int8_t, c_int32_t, c_ptr implicit none - interface - -#if (__SSE3__ && __SSE4_1__ && __SSE4_2__) -#if (__AVX__ && __AVX2__ && __FMA__ && __BMI__ && __BMI2__) - - subroutine mm256_compress_block_bmi2(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm256_compress_block_bmi2") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine mm256_compress_block_avx2(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm256_compress_block_avx2") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - + integer(c_int), parameter :: PERNIX_BACKEND_AUTO = 0_c_int + integer(c_int), parameter :: PERNIX_BACKEND_FALLBACK = 1_c_int + integer(c_int), parameter :: PERNIX_STATUS_OK = 0_c_int - subroutine mm256_compress_blocks_bmi2(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm256_compress_blocks_bmi2") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - - subroutine mm256_compress_blocks_avx2(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm256_compress_blocks_avx2") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - -#if (__AVX512BW__ && __AVX512CD__ && __AVX512DQ__ && __AVX512F__ && __AVX512VL__ && __AVX512VBMI__) - - subroutine mm512_compress_block_avx512vbmi(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm512_compress_block_avx512vbmi") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine mm512_compress_blocks_avx512vbmi(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm512_compress_blocks_avx512vbmi") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - -#endif -#endif -#endif - - subroutine compress_block_fallback(bit_width, input_ptr, scale, output_ptr) bind(C, name = "compress_block_fallback") - import :: c_int8_t, c_float, c_ptr + interface + function pernix_compress_block_f32(backend, bit_width, block_size, input_ptr, scale, output_ptr) & + bind(C, name = "pernix_compress_block_f32") result(status) + import :: c_float, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr real(c_float), value :: scale type(c_ptr), value :: output_ptr - end subroutine + integer(c_int) :: status + end function - subroutine compress_blocks_fallback(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "compress_blocks_fallback") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_compress_blocks_f32(backend, bit_width, block_size, input_ptr, scale, output_ptr, blocks) & + bind(C, name = "pernix_compress_blocks_f32") result(status) + import :: c_float, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr real(c_float), value :: scale type(c_ptr), value :: output_ptr integer(c_int32_t), value :: blocks - end subroutine + integer(c_int) :: status + end function - subroutine compress_block(bit_width, input_ptr, scale, output_ptr) bind(C, name = "compress_block") - import :: c_int8_t, c_float, c_ptr + function pernix_compress_block_f64(backend, bit_width, block_size, input_ptr, scale, output_ptr) & + bind(C, name = "pernix_compress_block_f64") result(status) + import :: c_double, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr - real(c_float), value :: scale + real(c_double), value :: scale type(c_ptr), value :: output_ptr - end subroutine + integer(c_int) :: status + end function - subroutine compress_blocks(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "compress_blocks") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_compress_blocks_f64(backend, bit_width, block_size, input_ptr, scale, output_ptr, blocks) & + bind(C, name = "pernix_compress_blocks_f64") result(status) + import :: c_double, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr - real(c_float), value :: scale + real(c_double), value :: scale type(c_ptr), value :: output_ptr integer(c_int32_t), value :: blocks - end subroutine - + integer(c_int) :: status + end function end interface - -end module pernix_compression \ No newline at end of file +end module pernix_compression diff --git a/bindings/fortran/src/decompression.f90 b/bindings/fortran/src/decompression.f90 index fb34e75..c2347fc 100644 --- a/bindings/fortran/src/decompression.f90 +++ b/bindings/fortran/src/decompression.f90 @@ -1,105 +1,60 @@ module pernix_decompression - use iso_c_binding, only : c_int8_t, c_int32_t, c_float, c_ptr + use iso_c_binding, only : c_bool, c_double, c_float, c_int, c_int8_t, c_int32_t, c_ptr implicit none interface - -#if (__SSE3__ && __SSE4_1__ && __SSE4_2__) -#if (__AVX__ && __AVX2__ && __FMA__ && __BMI__ && __BMI2__) - - subroutine mm256_decompress_block_bmi2(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm256_decompress_block_bmi2") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine mm256_decompress_block_avx2(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm256_decompress_block_avx2") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - - subroutine mm256_decompress_blocks_bmi2(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm256_decompress_blocks_bmi2") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_decompress_block_f32(backend, bit_width, block_size, input_ptr, scale, output_ptr, sign_values) & + bind(C, name = "pernix_decompress_block_f32") result(status) + import :: c_bool, c_float, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr real(c_float), value :: scale type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine + logical(c_bool), value :: sign_values + integer(c_int) :: status + end function - subroutine mm256_decompress_blocks_avx2(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm256_decompress_blocks_avx2") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_decompress_blocks_f32(backend, bit_width, block_size, input_ptr, scale, output_ptr, blocks, & + sign_values) bind(C, name = "pernix_decompress_blocks_f32") result(status) + import :: c_bool, c_float, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr real(c_float), value :: scale type(c_ptr), value :: output_ptr integer(c_int32_t), value :: blocks - end subroutine + logical(c_bool), value :: sign_values + integer(c_int) :: status + end function -#if (__AVX512BW__ && __AVX512CD__ && __AVX512DQ__ && __AVX512F__ && __AVX512VL__ && __AVX512VBMI__) - - subroutine mm512_decompress_block_avx512vbmi(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm512_decompress_block_avx512vbmi") - import :: c_int8_t, c_float, c_ptr + function pernix_decompress_block_f64(backend, bit_width, block_size, input_ptr, scale, output_ptr, sign_values) & + bind(C, name = "pernix_decompress_block_f64") result(status) + import :: c_bool, c_double, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr - real(c_float), value :: scale + real(c_double), value :: scale type(c_ptr), value :: output_ptr - end subroutine + logical(c_bool), value :: sign_values + integer(c_int) :: status + end function - subroutine mm512_decompress_blocks_avx512vbmi(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm512_decompress_blocks_avx512vbmi") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_decompress_blocks_f64(backend, bit_width, block_size, input_ptr, scale, output_ptr, blocks, & + sign_values) bind(C, name = "pernix_decompress_blocks_f64") result(status) + import :: c_bool, c_double, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr - real(c_float), value :: scale + real(c_double), value :: scale type(c_ptr), value :: output_ptr integer(c_int32_t), value :: blocks - end subroutine - -#endif -#endif -#endif - - subroutine decompress_block_fallback(bit_width, input_ptr, scale, output_ptr) bind(C, name = "decompress_block_fallback") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine decompress_blocks_fallback(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "decompress_blocks_fallback") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - - subroutine decompress_block(bit_width, input_ptr, scale, output_ptr) bind(C, name = "decompress_block") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine decompress_blocks(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "decompress_blocks") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - - + logical(c_bool), value :: sign_values + integer(c_int) :: status + end function end interface - -end module pernix_decompression \ No newline at end of file +end module pernix_decompression diff --git a/include/pernix/compat.h b/include/pernix/compat.h index 498c83a..b749f87 100644 --- a/include/pernix/compat.h +++ b/include/pernix/compat.h @@ -1,9 +1,13 @@ #ifndef PERNIX_COMPAT_H #define PERNIX_COMPAT_H -#include -#include +#if defined(__cplusplus) #include +#include +#else +#include +#include +#endif #ifndef __always_inline #if defined(__GNUC__) || defined(__clang__) @@ -37,8 +41,8 @@ typedef int16_t i16; typedef int32_t i32; typedef int64_t i64; -typedef float_t f32; -typedef double_t f64; +typedef float f32; +typedef double f64; typedef size_t usize; diff --git a/include/pernix/detail/api.hpp b/include/pernix/detail/api.hpp index 0a4df88..ede83cf 100644 --- a/include/pernix/detail/api.hpp +++ b/include/pernix/detail/api.hpp @@ -4,12 +4,51 @@ #include #include +#include +#include #include #include namespace pernix::detail { +constexpr u8 min_bit_width = 1; +constexpr u8 max_bit_width = 24; +constexpr u32 fixed_block_size = 64; + +constexpr bool is_valid_bit_width(const u8 bit_width) { + return bit_width >= min_bit_width && bit_width <= max_bit_width; +} + constexpr bool is_valid_block_size(const u32 block_size) { - return block_size == 64 || block_size == 128 || block_size == 256 || block_size == 512 || block_size == 1024; + return block_size == fixed_block_size || block_size == 128 || block_size == 256 || block_size == 512 || + block_size == 1024; +} + +constexpr u32 elements_per_block(const u8 bit_width, const u32 block_size = fixed_block_size) { + return is_valid_bit_width(bit_width) && is_valid_block_size(block_size) ? (block_size * 8U) / bit_width : 0; +} + +template +constexpr ScaleType quantization_levels(const u8 bit_width) { + return bit_width == 1 ? ScaleType{1} : static_cast((1U << (bit_width - 1U)) - 1U); +} + +template +inline pernix_status scale_from_bmax(const ScaleType bmax, const u8 bit_width, ScaleType *scale) { + if (scale == nullptr || !is_valid_bit_width(bit_width) || !std::isfinite(bmax) || bmax < ScaleType{0}) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + if (bmax <= ScaleType{0}) { + *scale = std::numeric_limits::epsilon(); + } else { + *scale = bmax / quantization_levels(bit_width); + } + return PERNIX_STATUS_OK; +} + +template +inline bool is_valid_scale(const ScaleType scale) { + return std::isfinite(scale) && scale > ScaleType{0}; } inline pernix_status select_error_status(const std::string_view kernel_name) { @@ -31,6 +70,9 @@ inline pernix_status compress_block(const Backend backend, const u8 bit_width, c if (!is_valid_block_size(block_size)) { return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } + if (!is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } const auto kernel = [&] { if constexpr (std::is_same_v) { @@ -52,9 +94,15 @@ inline pernix_status compress_blocks(const Backend backend, const u8 bit_width, if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } + if (blocks == 0) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } if (!is_valid_block_size(block_size)) { return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } + if (!is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } const auto kernel = [&] { if constexpr (std::is_same_v) { @@ -80,6 +128,9 @@ inline pernix_status decompress_block(const Backend backend, const u8 bit_width, if (!is_valid_block_size(block_size)) { return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } + if (!is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } const auto kernel = [&] { if constexpr (std::is_same_v) { @@ -102,9 +153,15 @@ inline pernix_status decompress_blocks(const Backend backend, const u8 bit_width if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } + if (blocks == 0) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } if (!is_valid_block_size(block_size)) { return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; } + if (!is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } const auto kernel = [&] { if constexpr (std::is_same_v) { diff --git a/include/pernix/pernix.h b/include/pernix/pernix.h index 5077a14..5b7ed8e 100644 --- a/include/pernix/pernix.h +++ b/include/pernix/pernix.h @@ -3,6 +3,10 @@ #include +#if !defined(__cplusplus) +#include +#endif + #if defined(__cplusplus) extern "C" { #endif @@ -30,6 +34,22 @@ typedef enum pernix_backend { PERNIX_BACKEND_FALLBACK_SIMD = 8 } pernix_backend; +PERNIX_API u8 pernix_min_bit_width(void); + +PERNIX_API u8 pernix_max_bit_width(void); + +PERNIX_API bool pernix_is_valid_bit_width(u8 bit_width); + +PERNIX_API bool pernix_is_valid_block_size(u32 block_size); + +PERNIX_API u32 pernix_compressed_block_size(void); + +PERNIX_API u32 pernix_elements_per_block(u8 bit_width); + +PERNIX_API pernix_status pernix_scale_f32(float bmax, u8 bit_width, float* scale); + +PERNIX_API pernix_status pernix_scale_f64(double bmax, u8 bit_width, double* scale); + PERNIX_API pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, void* output); diff --git a/include/pernix/pernix.hpp b/include/pernix/pernix.hpp index d0e7ac4..d07e152 100644 --- a/include/pernix/pernix.hpp +++ b/include/pernix/pernix.hpp @@ -4,46 +4,145 @@ #include #include +#include #include namespace pernix { +constexpr u8 min_bit_width() { + return detail::min_bit_width; +} + +constexpr u8 max_bit_width() { + return detail::max_bit_width; +} + +constexpr bool is_valid_bit_width(const u8 bit_width) { + return detail::is_valid_bit_width(bit_width); +} + +constexpr bool is_valid_block_size(const u32 block_size) { + return detail::is_valid_block_size(block_size); +} + +constexpr u32 compressed_block_size() { + return detail::fixed_block_size; +} + +constexpr u32 elements_per_block(const u8 bit_width) { + return detail::elements_per_block(bit_width); +} + +template +__always_inline int scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT &scale) { + return detail::scale_from_bmax(bmax, bit_width, &scale); +} + +namespace detail { +template +__always_inline int validate_compress_spans(const u8 bit_width, const u32 block_size, + const std::span input, const std::span output, + const u32 blocks) { + if (!is_valid_bit_width(bit_width)) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + if (blocks == 0) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + const usize required_input = static_cast(elements_per_block(bit_width, block_size)) * blocks; + const usize required_output = static_cast(block_size) * blocks; + return input.size() >= required_input && output.size() >= required_output ? PERNIX_STATUS_OK + : PERNIX_STATUS_INVALID_ARGUMENT; +} + +template +__always_inline int validate_decompress_spans(const u8 bit_width, const u32 block_size, const std::span input, + const std::span output, const u32 blocks) { + if (!is_valid_bit_width(bit_width)) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + if (blocks == 0) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + const usize required_input = static_cast(block_size) * blocks; + const usize required_output = static_cast(elements_per_block(bit_width, block_size)) * blocks; + return input.size() >= required_input && output.size() >= required_output ? PERNIX_STATUS_OK + : PERNIX_STATUS_INVALID_ARGUMENT; +} +} // namespace detail + __always_inline int compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const float scale, std::span output) { + if (const int status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); + status != PERNIX_STATUS_OK) { + return status; + } return detail::compress_block(backend, bit_width, block_size, input.data(), scale, output.data()); } __always_inline int compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const double scale, std::span output) { + if (const int status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); + status != PERNIX_STATUS_OK) { + return status; + } return detail::compress_block(backend, bit_width, block_size, input.data(), scale, output.data()); } __always_inline int decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const float scale, std::span output, const bool sign_values = true) { + if (const int status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); + status != PERNIX_STATUS_OK) { + return status; + } return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } __always_inline int decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const double scale, std::span output, const bool sign_values = true) { + if (const int status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); + status != PERNIX_STATUS_OK) { + return status; + } return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } __always_inline int compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const float scale, std::span output, const u32 blocks) { + if (const int status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); + status != PERNIX_STATUS_OK) { + return status; + } return detail::compress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks); } __always_inline int compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const double scale, std::span output, const u32 blocks) { + if (const int status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); + status != PERNIX_STATUS_OK) { + return status; + } return detail::compress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks); } __always_inline int decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const float scale, std::span output, const u32 blocks, const bool sign_values = true) { + if (const int status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); + status != PERNIX_STATUS_OK) { + return status; + } return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, sign_values); } @@ -51,6 +150,10 @@ __always_inline int decompress_blocks(Backend backend, const u8 bit_width, const __always_inline int decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, const double scale, std::span output, const u32 blocks, const bool sign_values = true) { + if (const int status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); + status != PERNIX_STATUS_OK) { + return status; + } return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, sign_values); } diff --git a/src/pernix.cpp b/src/pernix.cpp index db8ab05..7cccbe8 100644 --- a/src/pernix.cpp +++ b/src/pernix.cpp @@ -1,6 +1,38 @@ #include extern "C" { +u8 pernix_min_bit_width(void) { + return pernix::detail::min_bit_width; +} + +u8 pernix_max_bit_width(void) { + return pernix::detail::max_bit_width; +} + +bool pernix_is_valid_bit_width(u8 bit_width) { + return pernix::detail::is_valid_bit_width(bit_width); +} + +bool pernix_is_valid_block_size(u32 block_size) { + return pernix::detail::is_valid_block_size(block_size); +} + +u32 pernix_compressed_block_size(void) { + return pernix::detail::fixed_block_size; +} + +u32 pernix_elements_per_block(u8 bit_width) { + return pernix::detail::elements_per_block(bit_width); +} + +pernix_status pernix_scale_f32(float bmax, u8 bit_width, float *scale) { + return pernix::detail::scale_from_bmax(bmax, bit_width, scale); +} + +pernix_status pernix_scale_f64(double bmax, u8 bit_width, double *scale) { + return pernix::detail::scale_from_bmax(bmax, bit_width, scale); +} + pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, float scale, void *output) { return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a2803dc..ee50e83 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -31,6 +31,11 @@ endforeach () add_custom_target(pernix_tests DEPENDS ${PERNIX_TEST_TARGETS}) +add_executable(pernix_c_api_roundtrip c_api_roundtrip.c) +target_link_libraries(pernix_c_api_roundtrip PRIVATE pernix) +target_include_directories(pernix_c_api_roundtrip PRIVATE ${PROJECT_SOURCE_DIR}/include) +add_test(NAME pernix_c_api_roundtrip COMMAND pernix_c_api_roundtrip) + include(GoogleTest) foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) gtest_discover_tests(pernix_tests_${BLOCK_SIZE} diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp index 75f91ac..54dd0a6 100644 --- a/tests/fallback_tests.cpp +++ b/tests/fallback_tests.cpp @@ -457,6 +457,74 @@ TEST(FallbackEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { EXPECT_EQ(restored[2], 0.0f); } +TEST(FallbackEdgeTest, SignValuesFalseTreatsPackedValuesAsUnsigned) { + constexpr u32 BS = 64; + const std::array input{0x0F}; + + std::array signed_output{}; + std::array unsigned_output{}; + + auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, signed_output.data(), + true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, unsigned_output.data(), + false); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(signed_output[0], -1.0f); + EXPECT_EQ(unsigned_output[0], 15.0f); +} + +TEST(FallbackEdgeTest, CApiRejectsInvalidScale) { + constexpr u32 BS = 64; + constexpr u32 BW = 8; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + std::array compressed{}; + std::array output{}; + + EXPECT_EQ(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 0.0f, compressed.data()), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), -1.0f, output.data(), + true), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), + std::numeric_limits::infinity(), compressed.data()), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(FallbackEdgeTest, CApiRejectsInvalidScaleF64) { + constexpr u32 BS = 64; + constexpr u32 BW = 8; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + std::array compressed{}; + std::array output{}; + + EXPECT_EQ(pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 0.0, compressed.data()), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), -1.0, output.data(), + true), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(FallbackEdgeTest, ScaleHelpersValidateInputs) { + float scale_f32 = 0.0f; + double scale_f64 = 0.0; + + EXPECT_EQ(pernix_scale_f32(32767.0f, 16, &scale_f32), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(scale_f32, 1.0f); + EXPECT_EQ(pernix_scale_f64(32767.0, 16, &scale_f64), PERNIX_STATUS_OK); + EXPECT_DOUBLE_EQ(scale_f64, 1.0); + EXPECT_EQ(pernix_scale_f32(1.0f, 0, &scale_f32), PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_scale_f32(-1.0f, 16, &scale_f32), PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_scale_f64(std::numeric_limits::infinity(), 16, &scale_f64), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_scale_f64(1.0, 16, nullptr), PERNIX_STATUS_INVALID_ARGUMENT); +} + #if defined(PERNIX_BUILD_FALLBACK_STDPAR) TEST(FallbackStdparEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { constexpr u32 BS = 64; diff --git a/tests/header_only_tests.cpp b/tests/header_only_tests.cpp index 77e900d..240048c 100644 --- a/tests/header_only_tests.cpp +++ b/tests/header_only_tests.cpp @@ -3,6 +3,7 @@ #include #include +#include #include namespace { @@ -42,6 +43,91 @@ TEST(HeaderOnlyPernix, FallbackFloatRoundTrip) { expect_round_trip(pernix::Backend::Fallback); } +TEST(HeaderOnlyPernix, PublicHelpersDescribeFixedBlockFormat) { + EXPECT_EQ(pernix::min_bit_width(), 1); + EXPECT_EQ(pernix::max_bit_width(), 24); + EXPECT_TRUE(pernix::is_valid_bit_width(1)); + EXPECT_TRUE(pernix::is_valid_bit_width(24)); + EXPECT_FALSE(pernix::is_valid_bit_width(0)); + EXPECT_FALSE(pernix::is_valid_bit_width(25)); + EXPECT_TRUE(pernix::is_valid_block_size(64)); + EXPECT_FALSE(pernix::is_valid_block_size(96)); + EXPECT_EQ(pernix::compressed_block_size(), 64); + EXPECT_EQ(pernix::elements_per_block(16), 32); + EXPECT_EQ(pernix::elements_per_block(0), 0); + + float scale_f32 = 0.0f; + EXPECT_EQ(pernix::scale_from_bmax(32767.0f, 16, scale_f32), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(scale_f32, 1.0f); + + double scale_f64 = 0.0; + EXPECT_EQ(pernix::scale_from_bmax(0.0, 16, scale_f64), PERNIX_STATUS_OK); + EXPECT_GT(scale_f64, 0.0); +} + +TEST(HeaderOnlyPernix, RejectsUndersizedCompressInputSpan) { + std::vector input(kBlockElements - 1U, 0.0f); + std::vector output(kBlockSize); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), + 1.0f, std::span(output)), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, RejectsUndersizedCompressOutputSpan) { + std::vector input(kBlockElements, 0.0f); + std::vector output(kBlockSize - 1U); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), + 1.0f, std::span(output)), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, RejectsUndersizedDecompressSpans) { + std::vector short_input(kBlockSize - 1U); + std::vector output(kBlockElements); + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, + std::span(short_input), 1.0f, std::span(output)), + PERNIX_STATUS_INVALID_ARGUMENT); + + std::vector input(kBlockSize); + std::vector short_output(kBlockElements - 1U); + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), + 1.0f, std::span(short_output)), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, RejectsInvalidSpanParametersBeforeDispatch) { + std::vector input(kBlockElements, 0.0f); + std::vector output(kBlockSize); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, 0, kBlockSize, std::span(input), 1.0f, + std::span(output)), + PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, 96, std::span(input), 1.0f, + std::span(output)), + PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); + EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, kBitWidth, kBlockSize, + std::span(input), 1.0f, std::span(output), 0), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, RejectsInvalidScale) { + std::vector input(kBlockElements, 0.0f); + std::vector compressed(kBlockSize); + std::vector output(input.size()); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), + 0.0f, std::span(compressed)), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, + std::span(compressed), -1.0f, std::span(output)), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), + std::numeric_limits::infinity(), std::span(compressed)), + PERNIX_STATUS_INVALID_ARGUMENT); +} + TEST(HeaderOnlyPernix, FallbackAliasMatchesScalar) { EXPECT_EQ(static_cast(pernix::Backend::Fallback), static_cast(pernix::Backend::FallbackScalar)); } From 9d373b15caf664d629b864308e93e94403bdafcd Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Wed, 8 Jul 2026 21:13:07 +0200 Subject: [PATCH 19/43] feat: add roundtrip examples in C, C++, and Fortran for compression and decompression --- examples/CMakeLists.txt | 10 +++++++ examples/c/roundtrip.c | 38 ++++++++++++++++++++++++ examples/cpp/roundtrip.cpp | 45 +++++++++++++++++++++++++++++ examples/fortran/roundtrip.f90 | 32 ++++++++++++++++++++ tests/c_api_roundtrip.c | 53 ++++++++++++++++++++++++++++++++++ 5 files changed, 178 insertions(+) create mode 100644 examples/CMakeLists.txt create mode 100644 examples/c/roundtrip.c create mode 100644 examples/cpp/roundtrip.cpp create mode 100644 examples/fortran/roundtrip.f90 create mode 100644 tests/c_api_roundtrip.c diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 0000000..5021306 --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,10 @@ +add_executable(pernix_example_cpp cpp/roundtrip.cpp) +target_link_libraries(pernix_example_cpp PRIVATE pernix) + +add_executable(pernix_example_c c/roundtrip.c) +target_link_libraries(pernix_example_c PRIVATE pernix) + +if (PERNIX_ENABLE_FORTRAN_BINDINGS) + add_executable(pernix_example_fortran fortran/roundtrip.f90) + target_link_libraries(pernix_example_fortran PRIVATE pernix_bindings_fortran pernix) +endif () diff --git a/examples/c/roundtrip.c b/examples/c/roundtrip.c new file mode 100644 index 0000000..b4d58fa --- /dev/null +++ b/examples/c/roundtrip.c @@ -0,0 +1,38 @@ +#include + +int main(void) { + enum { bit_width = 16, block_size = 64, elements = (block_size * 8) / bit_width }; + float input[elements]; + float restored[elements]; + u8 compressed[block_size]; + float bmax = 0.0f; + + for (int i = 0; i < elements; ++i) { + input[i] = ((float)i - 16.0f) * 0.125f; + const float magnitude = input[i] < 0.0f ? -input[i] : input[i]; + bmax = bmax < magnitude ? magnitude : bmax; + } + + float scale = 0.0f; + if (pernix_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK) { + return 1; + } + + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 1.0f / scale, + compressed) != PERNIX_STATUS_OK) { + return 1; + } + if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, + true) != PERNIX_STATUS_OK) { + return 2; + } + + for (int i = 0; i < elements; ++i) { + const float diff = restored[i] - input[i]; + const float error = diff < 0.0f ? -diff : diff; + if (error > scale) { + return 3; + } + } + return 0; +} diff --git a/examples/cpp/roundtrip.cpp b/examples/cpp/roundtrip.cpp new file mode 100644 index 0000000..2393493 --- /dev/null +++ b/examples/cpp/roundtrip.cpp @@ -0,0 +1,45 @@ +#include + +#include +#include +#include +#include + +int main() { + constexpr u8 bit_width = 16; + constexpr u32 block_size = 64; + constexpr std::size_t elements = (block_size * 8U) / bit_width; + + std::array input{}; + for (std::size_t i = 0; i < input.size(); ++i) { + input[i] = std::sin(static_cast(i) * 0.25f); + } + + float bmax = 0.0f; + for (const float value : input) { + bmax = std::max(bmax, std::abs(value)); + } + float scale = 0.0f; + if (pernix::scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK) { + return 1; + } + + std::array compressed{}; + std::array restored{}; + + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, 1.0f / scale, compressed) != + PERNIX_STATUS_OK) { + return 1; + } + if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, scale, restored) != + PERNIX_STATUS_OK) { + return 2; + } + + for (std::size_t i = 0; i < input.size(); ++i) { + if (std::abs(restored[i] - input[i]) > scale) { + return 3; + } + } + return 0; +} diff --git a/examples/fortran/roundtrip.f90 b/examples/fortran/roundtrip.f90 new file mode 100644 index 0000000..91ac598 --- /dev/null +++ b/examples/fortran/roundtrip.f90 @@ -0,0 +1,32 @@ +program pernix_example_fortran + use iso_c_binding, only : c_bool, c_float, c_int, c_int8_t, c_int32_t, c_loc + use pernix_compression + use pernix_decompression + implicit none + + integer(c_int8_t), parameter :: bit_width = 16_c_int8_t + integer(c_int32_t), parameter :: block_size = 64_c_int32_t + integer, parameter :: elements = (64 * 8) / 16 + real(c_float), target :: input(elements) + real(c_float), target :: restored(elements) + integer(c_int8_t), target :: compressed(block_size) + real(c_float) :: bmax + real(c_float) :: scale + integer(c_int) :: status + integer :: i + + do i = 1, elements + input(i) = sin(real(i, c_float) * 0.25_c_float) + end do + + bmax = maxval(abs(input)) + scale = bmax / real((2 ** (int(bit_width) - 1)) - 1, c_float) + + status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(input), & + 1.0_c_float / scale, c_loc(compressed)) + if (status /= PERNIX_STATUS_OK) stop 1 + + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(compressed), & + scale, c_loc(restored), .true._c_bool) + if (status /= PERNIX_STATUS_OK) stop 2 +end program pernix_example_fortran diff --git a/tests/c_api_roundtrip.c b/tests/c_api_roundtrip.c new file mode 100644 index 0000000..49ce232 --- /dev/null +++ b/tests/c_api_roundtrip.c @@ -0,0 +1,53 @@ +#include + +int main(void) { + enum { bit_width = 16, block_size = 64, elements = (block_size * 8) / bit_width }; + + if (pernix_min_bit_width() != 1 || pernix_max_bit_width() != 24) { + return 1; + } + if (!pernix_is_valid_bit_width(bit_width) || pernix_is_valid_bit_width(25)) { + return 1; + } + if (pernix_compressed_block_size() != 64 || pernix_elements_per_block(bit_width) != elements) { + return 1; + } + + float input[elements]; + float restored[elements]; + u8 compressed[block_size]; + float bmax = 0.0f; + + for (int i = 0; i < elements; ++i) { + input[i] = ((float)i - 16.0f) * 0.125f; + const float magnitude = input[i] < 0.0f ? -input[i] : input[i]; + bmax = bmax < magnitude ? magnitude : bmax; + } + + float scale = 0.0f; + if (pernix_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK) { + return 1; + } + + pernix_status status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, + 1.0f / scale, compressed); + if (status != PERNIX_STATUS_OK) { + return 1; + } + + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, + true); + if (status != PERNIX_STATUS_OK) { + return 2; + } + + for (int i = 0; i < elements; ++i) { + const float diff = restored[i] - input[i]; + const float error = diff < 0.0f ? -diff : diff; + if (error > scale) { + return 3; + } + } + + return 0; +} From 9169a9388b385c9d4af9394aaf3453e8dc6407fa Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Wed, 8 Jul 2026 21:30:12 +0200 Subject: [PATCH 20/43] feat: add CPU feature detection and caching for ARM and x86 architectures --- include/pernix/dispatch/cpu_features.h | 5 ++ include/pernix/dispatch/select.h | 2 + src/CMakeLists.txt | 118 +++++++++++++++++++------ src/dispatch/cpu_features_arm.cpp | 7 +- src/dispatch/cpu_features_x86.cpp | 16 +++- 5 files changed, 116 insertions(+), 32 deletions(-) diff --git a/include/pernix/dispatch/cpu_features.h b/include/pernix/dispatch/cpu_features.h index fc7354e..c25b7c9 100644 --- a/include/pernix/dispatch/cpu_features.h +++ b/include/pernix/dispatch/cpu_features.h @@ -36,6 +36,10 @@ struct CpuFeatures { bool sve2 = false; }; +#if defined(PERNIX_BUILD_LIB) +CpuFeatures detect_cpu_features(); +CpuFeatures get_cached_cpu_features(); +#else inline CpuFeatures detect_cpu_features() { CpuFeatures features{}; @@ -108,6 +112,7 @@ inline CpuFeatures get_cached_cpu_features() { static const CpuFeatures features = detect_cpu_features(); return features; } +#endif } #endif //PERNIX_CPU_FEATURES_H diff --git a/include/pernix/dispatch/select.h b/include/pernix/dispatch/select.h index b9deef0..33ed123 100644 --- a/include/pernix/dispatch/select.h +++ b/include/pernix/dispatch/select.h @@ -258,6 +258,8 @@ namespace pernix::internal { #endif } +#if !defined(PERNIX_BUILD_LIB) #include +#endif #endif //PERNIX_SELECT_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7ada82a..f6f449a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,7 +5,22 @@ add_library(pernix SHARED) add_library(pernix_header_only INTERFACE) add_library(pernix::pernix_header_only ALIAS pernix_header_only) -target_sources(pernix PRIVATE pernix.cpp) +set(PERNIX_GENERIC_SOURCE_FILES + pernix.cpp + dispatch/select.cpp + fallback/fallback_compression.cpp + fallback/fallback_decompression.cpp +) + +if (PERNIX_TARGET_IS_X86) + list(APPEND PERNIX_GENERIC_SOURCE_FILES dispatch/cpu_features_x86.cpp) +elseif (PERNIX_TARGET_IS_ARM64) + list(APPEND PERNIX_GENERIC_SOURCE_FILES dispatch/cpu_features_arm.cpp) +else () + list(APPEND PERNIX_GENERIC_SOURCE_FILES dispatch/cpu_features_x86.cpp) +endif () + +target_sources(pernix PRIVATE ${PERNIX_GENERIC_SOURCE_FILES}) target_compile_features(pernix PUBLIC cxx_std_23) target_compile_features(pernix_header_only INTERFACE cxx_std_23) target_compile_options(pernix PRIVATE ${PERNIX_PRIVATE_COMPILE_OPTIONS}) @@ -30,6 +45,18 @@ target_include_directories(pernix_header_only target_compile_definitions(pernix PRIVATE PERNIX_BUILD_LIB=1) +function(pernix_set_source_options) + set(options) + set(one_value_args) + set(multi_value_args SOURCES OPTIONS) + cmake_parse_arguments(PERNIX_SOURCE_OPTIONS "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + foreach (PERNIX_SOURCE_FILE IN LISTS PERNIX_SOURCE_OPTIONS_SOURCES) + set_source_files_properties("${PERNIX_SOURCE_FILE}" PROPERTIES + COMPILE_OPTIONS "${PERNIX_SOURCE_OPTIONS_OPTIONS}" + ) + endforeach () +endfunction() + if (PERNIX_ENABLE_LTO) set_target_properties(pernix PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) endif () @@ -41,61 +68,94 @@ endif () if (PERNIX_ENABLE_FALLBACK_STDPAR) find_package(TBB REQUIRED) - target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_STDPAR=1) target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_STDPAR=1) target_link_libraries(pernix PUBLIC TBB::tbb) target_link_libraries(pernix_header_only INTERFACE TBB::tbb) endif () if (PERNIX_ENABLE_FALLBACK_SIMD) - target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_SIMD=1) target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_SIMD=1) endif () if (PERNIX_ENABLE_X86_BMI2 AND PERNIX_TARGET_IS_X86) target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_BMI2=1) + target_sources(pernix PRIVATE + x86/bmi2/bmi2_compression.cpp + x86/bmi2/bmi2_decompression.cpp + ) + if (MSVC) + pernix_set_source_options(SOURCES + x86/bmi2/bmi2_compression.cpp + x86/bmi2/bmi2_decompression.cpp + OPTIONS /arch:AVX2) + else () + pernix_set_source_options(SOURCES + x86/bmi2/bmi2_compression.cpp + x86/bmi2/bmi2_decompression.cpp + OPTIONS -mavx2 -mbmi -mbmi2) + endif () endif () if (PERNIX_ENABLE_X86_AVX2 AND PERNIX_TARGET_IS_X86) target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_AVX2=1) + target_sources(pernix PRIVATE + x86/avx2/avx2_compression.cpp + x86/avx2/avx2_decompression.cpp + ) + if (MSVC) + pernix_set_source_options(SOURCES + x86/avx2/avx2_compression.cpp + x86/avx2/avx2_decompression.cpp + OPTIONS /arch:AVX2) + else () + pernix_set_source_options(SOURCES + x86/avx2/avx2_compression.cpp + x86/avx2/avx2_decompression.cpp + OPTIONS -mavx2) + endif () endif () if (PERNIX_ENABLE_X86_AVX512VBMI AND PERNIX_TARGET_IS_X86) target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_AVX512_VBMI=1) + target_sources(pernix PRIVATE + x86/avx512vbmi/avx512vbmi_compression.cpp + x86/avx512vbmi/avx512vbmi_decompression.cpp + ) + if (MSVC) + pernix_set_source_options(SOURCES + x86/avx512vbmi/avx512vbmi_compression.cpp + x86/avx512vbmi/avx512vbmi_decompression.cpp + OPTIONS /arch:AVX512) + else () + pernix_set_source_options(SOURCES + x86/avx512vbmi/avx512vbmi_compression.cpp + x86/avx512vbmi/avx512vbmi_decompression.cpp + OPTIONS -mavx -mbmi -mbmi2 -mavx2 -mavx512f -mavx512bw -mavx512vl -mavx512dq -mavx512cd -mavx512vbmi) + endif () endif () if (PERNIX_ENABLE_ARM64_NEON AND PERNIX_TARGET_IS_ARM64) target_compile_definitions(pernix PRIVATE PERNIX_BUILD_ARM64_NEON=1) + target_sources(pernix PRIVATE + arm64/neon/compression.cpp + arm64/neon/decompression.cpp + ) + pernix_set_source_options(SOURCES + arm64/neon/compression.cpp + arm64/neon/decompression.cpp + OPTIONS -march=armv8-a+simd) endif () if (PERNIX_ENABLE_ARM64_SVE2 AND PERNIX_TARGET_IS_ARM64) target_compile_definitions(pernix PRIVATE PERNIX_BUILD_ARM64_SVE2=1) -endif () - -if (PERNIX_TARGET_IS_X86) - if (MSVC) - if (PERNIX_ENABLE_X86_AVX512VBMI) - target_compile_options(pernix PRIVATE /arch:AVX512) - elseif (PERNIX_ENABLE_X86_AVX2 OR PERNIX_ENABLE_X86_BMI2) - target_compile_options(pernix PRIVATE /arch:AVX2) - endif () - else () - if (PERNIX_ENABLE_X86_AVX512VBMI) - target_compile_options(pernix PRIVATE -mavx -mbmi -mbmi2 -mavx2 -mavx512f -mavx512bw -mavx512vl -mavx512dq -mavx512cd -mavx512vbmi) - elseif (PERNIX_ENABLE_X86_BMI2) - target_compile_options(pernix PRIVATE -mbmi -mbmi2 -mavx2) - elseif (PERNIX_ENABLE_X86_AVX2) - target_compile_options(pernix PRIVATE -mavx2) - endif () - endif () -endif () - -if (PERNIX_TARGET_IS_ARM64) - if (PERNIX_ENABLE_ARM64_SVE2) - target_compile_options(pernix PRIVATE -march=armv8.2-a+simd+sve2) - elseif (PERNIX_ENABLE_ARM64_NEON) - target_compile_options(pernix PRIVATE -march=armv8-a+simd) - endif () + target_sources(pernix PRIVATE + arm64/sve2/compression.cpp + arm64/sve2/decompression.cpp + ) + pernix_set_source_options(SOURCES + arm64/sve2/compression.cpp + arm64/sve2/decompression.cpp + OPTIONS -march=armv8.2-a+simd+sve2) endif () if (PERNIX_DISABLE_BMI2) diff --git a/src/dispatch/cpu_features_arm.cpp b/src/dispatch/cpu_features_arm.cpp index 525962b..b31cf8b 100644 --- a/src/dispatch/cpu_features_arm.cpp +++ b/src/dispatch/cpu_features_arm.cpp @@ -31,4 +31,9 @@ CpuFeatures detect_cpu_features() { return features; } -} \ No newline at end of file + +CpuFeatures get_cached_cpu_features() { + static const CpuFeatures features = detect_cpu_features(); + return features; +} +} diff --git a/src/dispatch/cpu_features_x86.cpp b/src/dispatch/cpu_features_x86.cpp index 973df4d..9abe896 100644 --- a/src/dispatch/cpu_features_x86.cpp +++ b/src/dispatch/cpu_features_x86.cpp @@ -30,7 +30,10 @@ void cpuid(int out[4], int leaf, int subleaf) { } u64 xgetbv(unsigned int index) { - return _xgetbv(index); + u32 eax = 0; + u32 edx = 0; + __asm__ volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(index)); + return static_cast(eax) | (static_cast(edx) << 32U); } #endif @@ -82,6 +85,11 @@ CpuFeatures detect_cpu_features() { return features; } + +CpuFeatures get_cached_cpu_features() { + static const CpuFeatures features = detect_cpu_features(); + return features; +} } // namespace pernix::internal #else @@ -89,7 +97,11 @@ namespace pernix::internal { CpuFeatures detect_cpu_features() { return {}; } + +CpuFeatures get_cached_cpu_features() { + static const CpuFeatures features = detect_cpu_features(); + return features; +} } // namespace pernix::internal #endif - From 7341844a77a86f6b9625b3776001c4176679f46c Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Wed, 8 Jul 2026 21:51:41 +0200 Subject: [PATCH 21/43] feat: add fallback packing function for AVX2 and enhance unpacking logic --- include/pernix/x86/avx2/avx2_compression.h | 23 ++++++++++++++++------ include/pernix/x86/avx512vbmi/unpacking.h | 18 +++++++++++++++++ tests/CMakeLists.txt | 5 +++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/include/pernix/x86/avx2/avx2_compression.h b/include/pernix/x86/avx2/avx2_compression.h index 4e611d7..d2670e2 100644 --- a/include/pernix/x86/avx2/avx2_compression.h +++ b/include/pernix/x86/avx2/avx2_compression.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -422,6 +423,19 @@ __m256i mm256_pack_epi32_avx2(const __m256i& input) { } return _mm256_setzero_si256(); } + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +void store_packed_epi32_fallback_8(const __m256i& input, u8* __restrict__ output) { + alignas(32) std::array lanes{}; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(lanes.data()), input); + + std::vector block_values(lanes.size()); + for (usize i = 0; i < lanes.size(); ++i) { + block_values[i] = static_cast(lanes[i]); + } + pack_epi32_fallback(block_values, output); +} } // namespace internal /** @@ -456,8 +470,7 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scal const __m256 source = _mm256_loadu_ps(input); const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); - const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); - std::memcpy(output, &packed, BIT_WIDTH); + internal::store_packed_epi32_fallback_8(packed_input, output); input += 8; output += BIT_WIDTH; @@ -513,10 +526,8 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scal const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); __m256i combined = _mm256_castsi128_si256(quantized1); combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_avx2( - internal::mm256_clamp_signed_epi32(combined)); - // _mm_storeu_si128(reinterpret_cast<__m128i*>(output), _mm256_castsi256_si128(packed)); - std::memcpy(output, &packed, BIT_WIDTH); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(combined); + internal::store_packed_epi32_fallback_8(packed_input, output); input += 8; output += BIT_WIDTH; } diff --git a/include/pernix/x86/avx512vbmi/unpacking.h b/include/pernix/x86/avx512vbmi/unpacking.h index 2f6579a..647d3ad 100644 --- a/include/pernix/x86/avx512vbmi/unpacking.h +++ b/include/pernix/x86/avx512vbmi/unpacking.h @@ -83,6 +83,9 @@ __always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i& input) { values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0303)); + if constexpr (SIGN_VALUES) { + values_shift0 = _mm_srai_epi8(_mm_slli_epi8(values_shift0, 6), 6); + } return values_shift0; } else if constexpr (BIT_WIDTH == 4) { __m128i values_shift0 = input; @@ -95,6 +98,9 @@ __always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i& input) { values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0F0F)); + if constexpr (SIGN_VALUES) { + values_shift0 = _mm_srai_epi8(_mm_slli_epi8(values_shift0, 4), 4); + } return values_shift0; } else { using tables = unpack_tables_avx512_8; @@ -247,6 +253,9 @@ __always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i& input) values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0303)); + if constexpr (SIGN_VALUES) { + values_shift0 = _mm256_srai_epi8(_mm256_slli_epi8(values_shift0, 6), 6); + } return values_shift0; } else if constexpr (BIT_WIDTH == 4) { __m256i values_shift0 = input; @@ -259,6 +268,9 @@ __always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i& input) values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0F0F)); + if constexpr (SIGN_VALUES) { + values_shift0 = _mm256_srai_epi8(_mm256_slli_epi8(values_shift0, 4), 4); + } return values_shift0; } else { using tables = unpack_tables_avx512_8; @@ -411,6 +423,9 @@ __always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i& input) values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0303)); + if constexpr (SIGN_VALUES) { + values_shift0 = _mm512_srai_epi8(_mm512_slli_epi8(values_shift0, 6), 6); + } return values_shift0; } else if constexpr (BIT_WIDTH == 4) { __m512i values_shift0 = input; @@ -423,6 +438,9 @@ __always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i& input) values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0F0F)); + if constexpr (SIGN_VALUES) { + values_shift0 = _mm512_srai_epi8(_mm512_slli_epi8(values_shift0, 4), 4); + } return values_shift0; } else { using tables = unpack_tables_avx512_8; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ee50e83..18f7271 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,6 +36,11 @@ target_link_libraries(pernix_c_api_roundtrip PRIVATE pernix) target_include_directories(pernix_c_api_roundtrip PRIVATE ${PROJECT_SOURCE_DIR}/include) add_test(NAME pernix_c_api_roundtrip COMMAND pernix_c_api_roundtrip) +add_executable(pernix_c_api_validation c_api_validation.c) +target_link_libraries(pernix_c_api_validation PRIVATE pernix) +target_include_directories(pernix_c_api_validation PRIVATE ${PROJECT_SOURCE_DIR}/include) +add_test(NAME pernix_c_api_validation COMMAND pernix_c_api_validation) + include(GoogleTest) foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) gtest_discover_tests(pernix_tests_${BLOCK_SIZE} From 5c42e8d82472cb53065d0b2d8e51680c6a555fc2 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Wed, 8 Jul 2026 22:04:41 +0200 Subject: [PATCH 22/43] feat: introduce clearer scale helper functions for compression and decompression --- README.md | 37 +++++-- examples/c/roundtrip.c | 6 +- examples/cpp/roundtrip.cpp | 6 +- include/pernix/detail/api.hpp | 40 +++++++ include/pernix/pernix.h | 31 +++++- include/pernix/pernix.hpp | 196 +++++++++++++++++++--------------- src/pernix.cpp | 28 +++++ tests/header_only_tests.cpp | 26 +++++ 8 files changed, 269 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 6e6ed48..2f3cac2 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,16 @@ Decompression expects the forward scale, because decompression computes: output = quantized * scale ``` -In practice, compute `scale` from `bmax`, pass `1 / scale` to `compress_block`, and pass `scale` to -`decompress_block`. The helper functions `pernix_scale_f32`, `pernix_scale_f64`, and -`pernix::scale_from_bmax` compute this scale and return a small positive scale for `bmax == 0`. +In practice, compute the decompression scale from `bmax`, pass its inverse to `compress_block`, and pass the forward +scale to `decompress_block`. The clearer helper names are: + +* `pernix_decompression_scale_f32` / `pernix_decompression_scale_f64` +* `pernix_compression_scale_f32` / `pernix_compression_scale_f64` +* `pernix_inverse_scale_f32` / `pernix_inverse_scale_f64` +* `pernix::decompression_scale_from_bmax`, `pernix::compression_scale_from_bmax`, and `pernix::inverse_scale` + +The older `pernix_scale_f32`, `pernix_scale_f64`, and `pernix::scale_from_bmax` names are kept as compatibility aliases +for the forward decompression scale. Scale-from-`bmax` helpers return a small positive scale for `bmax == 0`. Scale arguments passed to compression and decompression must be finite and greater than zero. Passing zero, negative, NaN, or infinity returns `PERNIX_STATUS_INVALID_ARGUMENT`. @@ -56,6 +63,9 @@ The public helper APIs expose the fixed-format constants: * `pernix_compressed_block_size()` / `pernix::compressed_block_size()` return `64` * `pernix_elements_per_block(bit_width)` / `pernix::elements_per_block(bit_width)` return `(64 * 8) / bit_width`, or `0` for an invalid bit width +* `pernix_is_valid_bit_width(bit_width)` / `pernix::is_valid_bit_width(bit_width)` validate the public `1..24` range +* `pernix_is_valid_block_size(block_size)` / `pernix::is_valid_block_size(block_size)` validate the currently accepted + block sizes Input values should be finite for portable behavior. The scalar fallback clamps non-finite or out-of-range values before narrowing, but that behavior is not a cross-backend contract. @@ -71,12 +81,21 @@ The C ABI returns `pernix_status`: * `PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE`: unsupported block size * `PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION`: backend was requested but is not available for this build or CPU +Use `pernix_status_string(status)` or `pernix::status_string(status)` for diagnostic text. The misspelled historical +enumerator `PERNIX_STATUS_UNSUPPORTED_IMPLEMENTAION` remains as a deprecated source-compatibility alias for +`PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION`. + ## f32 and f64 APIs Use the `_f32` functions for `float` input/output and `_f64` functions for `double` input/output. Both variants pack to the same integer bit stream for the selected bit width. The scale type matches the floating-point type: `float` for `_f32`, `double` for `_f64`. +The C++ wrapper overloads return `pernix::Status`, an alias for the C ABI `pernix_status`. `std::span` arguments are +borrowed only for the duration of the call. For one 64-byte block, compression reads at least +`pernix::elements_per_block(bit_width)` input values and writes `pernix::compressed_block_size()` bytes. Multi-block +calls multiply those counts by `blocks`. + ## Compiling ```bash @@ -121,14 +140,16 @@ int main() { bmax = std::max(bmax, std::abs(value)); } float scale = 0.0f; - if (pernix::scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK) { + float inverse_scale = 0.0f; + if (pernix::decompression_scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK || + pernix::inverse_scale(scale, inverse_scale) != PERNIX_STATUS_OK) { return 1; } std::array compressed{}; std::array restored{}; - if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, 1.0f / scale, + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { return 1; } @@ -159,11 +180,13 @@ int main(void) { } float scale = 0.0f; - if (pernix_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK) { + float inverse_scale = 0.0f; + if (pernix_decompression_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK || + pernix_inverse_scale_f32(scale, &inverse_scale) != PERNIX_STATUS_OK) { return 1; } - if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 1.0f / scale, + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { return 1; } diff --git a/examples/c/roundtrip.c b/examples/c/roundtrip.c index b4d58fa..c4c4acd 100644 --- a/examples/c/roundtrip.c +++ b/examples/c/roundtrip.c @@ -14,11 +14,13 @@ int main(void) { } float scale = 0.0f; - if (pernix_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK) { + float inverse_scale = 0.0f; + if (pernix_decompression_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK || + pernix_inverse_scale_f32(scale, &inverse_scale) != PERNIX_STATUS_OK) { return 1; } - if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 1.0f / scale, + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { return 1; } diff --git a/examples/cpp/roundtrip.cpp b/examples/cpp/roundtrip.cpp index 2393493..69153ba 100644 --- a/examples/cpp/roundtrip.cpp +++ b/examples/cpp/roundtrip.cpp @@ -20,14 +20,16 @@ int main() { bmax = std::max(bmax, std::abs(value)); } float scale = 0.0f; - if (pernix::scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK) { + float inverse_scale = 0.0f; + if (pernix::decompression_scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK || + pernix::inverse_scale(scale, inverse_scale) != PERNIX_STATUS_OK) { return 1; } std::array compressed{}; std::array restored{}; - if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, 1.0f / scale, compressed) != + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { return 1; } diff --git a/include/pernix/detail/api.hpp b/include/pernix/detail/api.hpp index ede83cf..b540d41 100644 --- a/include/pernix/detail/api.hpp +++ b/include/pernix/detail/api.hpp @@ -51,6 +51,46 @@ inline bool is_valid_scale(const ScaleType scale) { return std::isfinite(scale) && scale > ScaleType{0}; } +template +inline pernix_status inverse_scale(const ScaleType scale, ScaleType *inverse_scale_value) { + if (inverse_scale_value == nullptr || !is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + *inverse_scale_value = ScaleType{1} / scale; + return PERNIX_STATUS_OK; +} + +template +inline pernix_status compression_scale_from_bmax(const ScaleType bmax, const u8 bit_width, + ScaleType *inverse_scale_value) { + ScaleType scale = ScaleType{0}; + const auto status = scale_from_bmax(bmax, bit_width, &scale); + if (status != PERNIX_STATUS_OK) { + return status; + } + return inverse_scale(scale, inverse_scale_value); +} + +inline const char *status_string(const pernix_status status) { + switch (status) { + case PERNIX_STATUS_OK: + return "PERNIX_STATUS_OK"; + case PERNIX_STATUS_INVALID_ARGUMENT: + return "PERNIX_STATUS_INVALID_ARGUMENT"; + case PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH: + return "PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH"; + case PERNIX_STATUS_UNSUPPORTED_BACKEND: + return "PERNIX_STATUS_UNSUPPORTED_BACKEND"; + case PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE: + return "PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE"; + case PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION: + return "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION"; + default: + return "PERNIX_STATUS_UNKNOWN"; + } +} + inline pernix_status select_error_status(const std::string_view kernel_name) { if (kernel_name == "invalid_backend") { return PERNIX_STATUS_UNSUPPORTED_BACKEND; diff --git a/include/pernix/pernix.h b/include/pernix/pernix.h index 5b7ed8e..51b9c75 100644 --- a/include/pernix/pernix.h +++ b/include/pernix/pernix.h @@ -18,7 +18,6 @@ typedef enum pernix_status { PERNIX_STATUS_UNSUPPORTED_BACKEND = -3, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE = -4, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION = -5, - PERNIX_STATUS_UNSUPPORTED_IMPLEMENTAION = PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION } pernix_status; typedef enum pernix_backend { @@ -46,17 +45,39 @@ PERNIX_API u32 pernix_compressed_block_size(void); PERNIX_API u32 pernix_elements_per_block(u8 bit_width); +PERNIX_API const char* pernix_status_string(pernix_status status); + +/* Computes the forward decompression scale from bmax and bit width. */ PERNIX_API pernix_status pernix_scale_f32(float bmax, u8 bit_width, float* scale); +/* Computes the forward decompression scale from bmax and bit width. */ PERNIX_API pernix_status pernix_scale_f64(double bmax, u8 bit_width, double* scale); +/* Clearer alias for pernix_scale_f32. */ +PERNIX_API pernix_status pernix_decompression_scale_f32(float bmax, u8 bit_width, float* scale); + +/* Clearer alias for pernix_scale_f64. */ +PERNIX_API pernix_status pernix_decompression_scale_f64(double bmax, u8 bit_width, double* scale); + +/* Computes 1 / scale for the compression API from a valid forward scale. */ +PERNIX_API pernix_status pernix_inverse_scale_f32(float scale, float* inverse_scale); + +/* Computes 1 / scale for the compression API from a valid forward scale. */ +PERNIX_API pernix_status pernix_inverse_scale_f64(double scale, double* inverse_scale); + +/* Computes the inverse compression scale directly from bmax and bit width. */ +PERNIX_API pernix_status pernix_compression_scale_f32(float bmax, u8 bit_width, float* inverse_scale); + +/* Computes the inverse compression scale directly from bmax and bit width. */ +PERNIX_API pernix_status pernix_compression_scale_f64(double bmax, u8 bit_width, double* inverse_scale); + PERNIX_API pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, - float scale, void* output); + float inverse_scale, void* output); PERNIX_API pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, - float scale, void* output, u32 blocks); + float inverse_scale, void* output, u32 blocks); PERNIX_API pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, @@ -68,11 +89,11 @@ PERNIX_API pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 PERNIX_API pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, - double scale, void* output); + double inverse_scale, void* output); PERNIX_API pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, - double scale, void* output, u32 blocks); + double inverse_scale, void* output, u32 blocks); PERNIX_API pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, diff --git a/include/pernix/pernix.hpp b/include/pernix/pernix.hpp index d07e152..c4f3c46 100644 --- a/include/pernix/pernix.hpp +++ b/include/pernix/pernix.hpp @@ -8,6 +8,8 @@ #include namespace pernix { +using Status = pernix_status; + constexpr u8 min_bit_width() { return detail::min_bit_width; } @@ -33,15 +35,35 @@ constexpr u32 elements_per_block(const u8 bit_width) { } template -__always_inline int scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT &scale) { +__always_inline Status scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT &scale) { return detail::scale_from_bmax(bmax, bit_width, &scale); } +template +__always_inline Status decompression_scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT &scale) { + return scale_from_bmax(bmax, bit_width, scale); +} + +template +__always_inline Status inverse_scale(const FloatT scale, FloatT &inverse_scale_value) { + return detail::inverse_scale(scale, &inverse_scale_value); +} + +template +__always_inline Status compression_scale_from_bmax(const FloatT bmax, const u8 bit_width, + FloatT &inverse_scale_value) { + return detail::compression_scale_from_bmax(bmax, bit_width, &inverse_scale_value); +} + +__always_inline const char *status_string(const Status status) { + return detail::status_string(status); +} + namespace detail { template -__always_inline int validate_compress_spans(const u8 bit_width, const u32 block_size, - const std::span input, const std::span output, - const u32 blocks) { +__always_inline Status validate_compress_spans(const u8 bit_width, const u32 block_size, + const std::span input, const std::span output, + const u32 blocks) { if (!is_valid_bit_width(bit_width)) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; } @@ -59,8 +81,9 @@ __always_inline int validate_compress_spans(const u8 bit_width, const u32 block_ } template -__always_inline int validate_decompress_spans(const u8 bit_width, const u32 block_size, const std::span input, - const std::span output, const u32 blocks) { +__always_inline Status validate_decompress_spans(const u8 bit_width, const u32 block_size, + const std::span input, const std::span output, + const u32 blocks) { if (!is_valid_bit_width(bit_width)) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; } @@ -78,68 +101,70 @@ __always_inline int validate_decompress_spans(const u8 bit_width, const u32 bloc } } // namespace detail -__always_inline int compress_block(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const float scale, std::span output) { - if (const int status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); +__always_inline Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const float inverse_scale, + std::span output) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } - return detail::compress_block(backend, bit_width, block_size, input.data(), scale, output.data()); + return detail::compress_block(backend, bit_width, block_size, input.data(), inverse_scale, output.data()); } -__always_inline int compress_block(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const double scale, std::span output) { - if (const int status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); +__always_inline Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const double inverse_scale, + std::span output) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } - return detail::compress_block(backend, bit_width, block_size, input.data(), scale, output.data()); + return detail::compress_block(backend, bit_width, block_size, input.data(), inverse_scale, output.data()); } -__always_inline int decompress_block(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const float scale, std::span output, - const bool sign_values = true) { - if (const int status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); +__always_inline Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const float scale, std::span output, + const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } -__always_inline int decompress_block(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const double scale, std::span output, - const bool sign_values = true) { - if (const int status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); +__always_inline Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const double scale, std::span output, + const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } -__always_inline int compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const float scale, std::span output, - const u32 blocks) { - if (const int status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); +__always_inline Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const float inverse_scale, + std::span output, const u32 blocks) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } - return detail::compress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks); + return detail::compress_blocks(backend, bit_width, block_size, input.data(), inverse_scale, output.data(), blocks); } -__always_inline int compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const double scale, std::span output, - const u32 blocks) { - if (const int status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); +__always_inline Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const double inverse_scale, + std::span output, const u32 blocks) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } - return detail::compress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks); + return detail::compress_blocks(backend, bit_width, block_size, input.data(), inverse_scale, output.data(), blocks); } -__always_inline int decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const float scale, std::span output, - const u32 blocks, const bool sign_values = true) { - if (const int status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); +__always_inline Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const float scale, std::span output, + const u32 blocks, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } @@ -147,10 +172,10 @@ __always_inline int decompress_blocks(Backend backend, const u8 bit_width, const sign_values); } -__always_inline int decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const double scale, std::span output, - const u32 blocks, const bool sign_values = true) { - if (const int status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); +__always_inline Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, + const std::span input, const double scale, std::span output, + const u32 blocks, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } @@ -159,89 +184,90 @@ __always_inline int decompress_blocks(Backend backend, const u8 bit_width, const } // convenience overloads without backend (defaults to Auto) -__always_inline int compress_block(const u8 bit_width, const u32 block_size, const std::span input, - const float scale, const std::span output) { - return compress_block(Backend::Auto, bit_width, block_size, input, scale, output); +__always_inline Status compress_block(const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, const std::span output) { + return compress_block(Backend::Auto, bit_width, block_size, input, inverse_scale, output); } -__always_inline int compress_block(const u8 bit_width, const u32 block_size, const std::span input, - const double scale, const std::span output) { - return compress_block(Backend::Auto, bit_width, block_size, input, scale, output); +__always_inline Status compress_block(const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, const std::span output) { + return compress_block(Backend::Auto, bit_width, block_size, input, inverse_scale, output); } -__always_inline int decompress_block(const u8 bit_width, const u32 block_size, const std::span input, - const float scale, const std::span output, const bool sign_values = true) { +__always_inline Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, + const float scale, const std::span output, + const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); } -__always_inline int decompress_block(const u8 bit_width, const u32 block_size, const std::span input, - const double scale, const std::span output, - const bool sign_values = true) { +__always_inline Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, + const double scale, const std::span output, + const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); } -__always_inline int compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, - const float scale, const std::span output, const u32 blocks) { - return compress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks); +__always_inline Status compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, const std::span output, const u32 blocks) { + return compress_blocks(Backend::Auto, bit_width, block_size, input, inverse_scale, output, blocks); } -__always_inline int compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, - const double scale, const std::span output, const u32 blocks) { - return compress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks); +__always_inline Status compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, const std::span output, const u32 blocks) { + return compress_blocks(Backend::Auto, bit_width, block_size, input, inverse_scale, output, blocks); } -__always_inline int decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, - const float scale, const std::span output, const u32 blocks, - const bool sign_values = true) { +__always_inline Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const float scale, const std::span output, const u32 blocks, + const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); } -__always_inline int decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, - const double scale, const std::span output, const u32 blocks, - const bool sign_values = true) { +__always_inline Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const double scale, const std::span output, const u32 blocks, + const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); } // convenience overloads without backend and without block_size (defaults to 64) -__always_inline int compress_block(const u8 bit_width, const std::span input, const float scale, - const std::span output) { - return compress_block(Backend::Auto, bit_width, 64, input, scale, output); +__always_inline Status compress_block(const u8 bit_width, const std::span input, + const float inverse_scale, const std::span output) { + return compress_block(Backend::Auto, bit_width, 64, input, inverse_scale, output); } -__always_inline int compress_block(const u8 bit_width, const std::span input, const double scale, - const std::span output) { - return compress_block(Backend::Auto, bit_width, 64, input, scale, output); +__always_inline Status compress_block(const u8 bit_width, const std::span input, + const double inverse_scale, const std::span output) { + return compress_block(Backend::Auto, bit_width, 64, input, inverse_scale, output); } -__always_inline int decompress_block(const u8 bit_width, const std::span input, const float scale, - const std::span output, const bool sign_values = true) { +__always_inline Status decompress_block(const u8 bit_width, const std::span input, const float scale, + const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); } -__always_inline int decompress_block(const u8 bit_width, const std::span input, const double scale, - const std::span output, const bool sign_values = true) { +__always_inline Status decompress_block(const u8 bit_width, const std::span input, const double scale, + const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); } -__always_inline int compress_blocks(const u8 bit_width, const std::span input, const float scale, - const std::span output, const u32 blocks) { - return compress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks); +__always_inline Status compress_blocks(const u8 bit_width, const std::span input, + const float inverse_scale, const std::span output, const u32 blocks) { + return compress_blocks(Backend::Auto, bit_width, 64, input, inverse_scale, output, blocks); } -__always_inline int compress_blocks(const u8 bit_width, const std::span input, const double scale, - const std::span output, const u32 blocks) { - return compress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks); +__always_inline Status compress_blocks(const u8 bit_width, const std::span input, + const double inverse_scale, const std::span output, const u32 blocks) { + return compress_blocks(Backend::Auto, bit_width, 64, input, inverse_scale, output, blocks); } -__always_inline int decompress_blocks(const u8 bit_width, const std::span input, const float scale, - const std::span output, const u32 blocks, - const bool sign_values = true) { +__always_inline Status decompress_blocks(const u8 bit_width, const std::span input, const float scale, + const std::span output, const u32 blocks, + const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); } -__always_inline int decompress_blocks(const u8 bit_width, const std::span input, const double scale, - const std::span output, const u32 blocks, - const bool sign_values = true) { +__always_inline Status decompress_blocks(const u8 bit_width, const std::span input, const double scale, + const std::span output, const u32 blocks, + const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); } } // namespace pernix diff --git a/src/pernix.cpp b/src/pernix.cpp index 7cccbe8..6adbd95 100644 --- a/src/pernix.cpp +++ b/src/pernix.cpp @@ -25,6 +25,10 @@ u32 pernix_elements_per_block(u8 bit_width) { return pernix::detail::elements_per_block(bit_width); } +const char *pernix_status_string(pernix_status status) { + return pernix::detail::status_string(status); +} + pernix_status pernix_scale_f32(float bmax, u8 bit_width, float *scale) { return pernix::detail::scale_from_bmax(bmax, bit_width, scale); } @@ -33,6 +37,30 @@ pernix_status pernix_scale_f64(double bmax, u8 bit_width, double *scale) { return pernix::detail::scale_from_bmax(bmax, bit_width, scale); } +pernix_status pernix_decompression_scale_f32(float bmax, u8 bit_width, float *scale) { + return pernix_scale_f32(bmax, bit_width, scale); +} + +pernix_status pernix_decompression_scale_f64(double bmax, u8 bit_width, double *scale) { + return pernix_scale_f64(bmax, bit_width, scale); +} + +pernix_status pernix_inverse_scale_f32(float scale, float *inverse_scale) { + return pernix::detail::inverse_scale(scale, inverse_scale); +} + +pernix_status pernix_inverse_scale_f64(double scale, double *inverse_scale) { + return pernix::detail::inverse_scale(scale, inverse_scale); +} + +pernix_status pernix_compression_scale_f32(float bmax, u8 bit_width, float *inverse_scale) { + return pernix::detail::compression_scale_from_bmax(bmax, bit_width, inverse_scale); +} + +pernix_status pernix_compression_scale_f64(double bmax, u8 bit_width, double *inverse_scale) { + return pernix::detail::compression_scale_from_bmax(bmax, bit_width, inverse_scale); +} + pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, float scale, void *output) { return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, diff --git a/tests/header_only_tests.cpp b/tests/header_only_tests.cpp index 240048c..3a229b2 100644 --- a/tests/header_only_tests.cpp +++ b/tests/header_only_tests.cpp @@ -65,6 +65,32 @@ TEST(HeaderOnlyPernix, PublicHelpersDescribeFixedBlockFormat) { EXPECT_GT(scale_f64, 0.0); } +TEST(HeaderOnlyPernix, ScaleHelpersNameCompressionAndDecompressionConventions) { + float decompression_scale = 0.0f; + float compression_scale = 0.0f; + float inverse_scale = 0.0f; + + EXPECT_EQ(pernix::decompression_scale_from_bmax(127.0f, 8, decompression_scale), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(decompression_scale, 1.0f); + EXPECT_EQ(pernix::compression_scale_from_bmax(127.0f, 8, compression_scale), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(compression_scale, 1.0f); + EXPECT_EQ(pernix::inverse_scale(decompression_scale, inverse_scale), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(inverse_scale, compression_scale); + + EXPECT_EQ(pernix::inverse_scale(0.0f, inverse_scale), PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::compression_scale_from_bmax(1.0f, 0, compression_scale), PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, StatusAliasAndStringsArePublic) { + pernix::Status status = PERNIX_STATUS_OK; + EXPECT_EQ(status, PERNIX_STATUS_OK); + EXPECT_STREQ(pernix::status_string(status), "PERNIX_STATUS_OK"); + EXPECT_EQ(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTAION, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); + EXPECT_STREQ(pernix::status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTAION), + "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION"); + EXPECT_STREQ(pernix::status_string(static_cast(123)), "PERNIX_STATUS_UNKNOWN"); +} + TEST(HeaderOnlyPernix, RejectsUndersizedCompressInputSpan) { std::vector input(kBlockElements - 1U, 0.0f); std::vector output(kBlockSize); From 527666d204701fdb072ad8c5855327e56879a278 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Wed, 8 Jul 2026 22:21:54 +0200 Subject: [PATCH 23/43] feat: refactor packing and unpacking functions to use std::span and improve clarity --- README.md | 4 +- include/pernix/dispatch/kernel.h | 2 +- include/pernix/fallback/scalar_compression.h | 77 ++++++++++++++----- .../pernix/fallback/scalar_decompression.h | 68 ++++++++++++---- include/pernix/x86/avx2/avx2_compression.h | 14 ++-- include/pernix/x86/avx2/avx2_tables.h | 2 +- include/pernix/x86/avx512vbmi/packing.h | 39 ++++++---- include/pernix/x86/avx512vbmi/unpacking.h | 18 ++--- tests/fallback_tests.cpp | 50 ++++++++++++ tests/header_only_tests.cpp | 5 +- 10 files changed, 206 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 2f3cac2..c6d0a4f 100644 --- a/README.md +++ b/README.md @@ -81,9 +81,7 @@ The C ABI returns `pernix_status`: * `PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE`: unsupported block size * `PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION`: backend was requested but is not available for this build or CPU -Use `pernix_status_string(status)` or `pernix::status_string(status)` for diagnostic text. The misspelled historical -enumerator `PERNIX_STATUS_UNSUPPORTED_IMPLEMENTAION` remains as a deprecated source-compatibility alias for -`PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION`. +Use `pernix_status_string(status)` or `pernix::status_string(status)` for diagnostic text. ## f32 and f64 APIs diff --git a/include/pernix/dispatch/kernel.h b/include/pernix/dispatch/kernel.h index b7c0618..47198c3 100644 --- a/include/pernix/dispatch/kernel.h +++ b/include/pernix/dispatch/kernel.h @@ -19,7 +19,7 @@ struct Kernel { return func != nullptr; } - Kernel(const std::string_view name, FuncType func) : name(name), func(func) { + Kernel(const std::string_view kernel_name, FuncType kernel_func) : name(kernel_name), func(kernel_func) { } }; } diff --git a/include/pernix/fallback/scalar_compression.h b/include/pernix/fallback/scalar_compression.h index e478826..9c30324 100644 --- a/include/pernix/fallback/scalar_compression.h +++ b/include/pernix/fallback/scalar_compression.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -54,7 +55,7 @@ __always_inline i32 clamp_signed_quantized(const i64 value) { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) -void pack_epi32_fallback_inner(const std::vector& input, const u8 bit_offset, +void pack_epi32_fallback_inner(const std::span input, const u8 bit_offset, u8* __restrict__ destination) { constexpr u32 bits_in_type = sizeof(T) * 8; constexpr u32 bitmask = BIT_WIDTH == bits_in_type @@ -86,7 +87,7 @@ void pack_epi32_fallback_inner(const std::vector& input, const u8 bit_offse template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -void pack_epi32_fallback(const std::vector& input, u8* __restrict__ destination) { +void pack_epi32_fallback(const std::span input, u8* __restrict__ destination) { if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return internal::pack_epi32_fallback_inner(input, 0, destination); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { @@ -95,6 +96,60 @@ void pack_epi32_fallback(const std::vector& input, u8* __restrict__ destina return internal::pack_epi32_fallback_inner(input, 0, destination); } } + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +void pack_epi32_fallback(const std::vector& input, u8* __restrict__ destination) { + return pack_epi32_fallback(std::span(input.data(), input.size()), destination); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v && + std::is_floating_point_v) +void quantize_and_pack_fallback_inner(const FloatT* __restrict__ input, const FloatT scale, + const u32 elements, const u8 bit_offset, + u8* __restrict__ destination) { + constexpr u32 bits_in_type = sizeof(T) * 8; + constexpr u32 bitmask = BIT_WIDTH == bits_in_type + ? std::numeric_limits::max() + : (1U << BIT_WIDTH) - 1U; + + std::size_t idx = 0; + std::size_t bits_in_buffer = bit_offset; + u64 buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; + +#pragma GCC unroll 64 + for (u32 i = 0; i < elements; i++) { + const i32 quantized = quantize_clamped(input[i], scale); + const u32 next_value = static_cast(quantized) & bitmask; + + buffer |= static_cast(next_value) << bits_in_buffer; + bits_in_buffer += BIT_WIDTH; + + while (bits_in_buffer >= 8) { + destination[idx++] = static_cast(buffer & 0xFFU); + buffer >>= 8; + bits_in_buffer -= 8; + } + } + + if (bits_in_buffer > 0) { + destination[idx] = static_cast(buffer & 0xFFU); + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +void quantize_and_pack_fallback(const FloatT* __restrict__ input, const FloatT scale, + const u32 elements, u8* __restrict__ destination) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); + } else { + return quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); + } +} } template @@ -108,14 +163,7 @@ int compress_block_fallback(const void* __restrict__ input_ptr, const f32 scale, std::memset(output, 0, BLOCK_SIZE); - std::vector block_values(elements_per_block); -#pragma GCC unroll 64 - for (u32 i = 0; i < elements_per_block; i++) { - const i32 quantized = internal::quantize_clamped(input[i], scale); - block_values[i] = static_cast(quantized); - } - - internal::pack_epi32_fallback(block_values, output); + internal::quantize_and_pack_fallback(input, scale, elements_per_block, output); return 0; } @@ -130,14 +178,7 @@ int compress_block_fallback(const void* __restrict__ input_ptr, const f64 scale, std::memset(output, 0, BLOCK_SIZE); - std::vector block_values(elements_per_block); -#pragma GCC unroll 32 - for (u32 i = 0; i < elements_per_block; i++) { - const i32 quantized = internal::quantize_clamped(input[i], scale); - block_values[i] = static_cast(quantized); - } - - internal::pack_epi32_fallback(block_values, output); + internal::quantize_and_pack_fallback(input, scale, elements_per_block, output); return 0; } diff --git a/include/pernix/fallback/scalar_decompression.h b/include/pernix/fallback/scalar_decompression.h index 1b2e68a..fc7bad0 100644 --- a/include/pernix/fallback/scalar_decompression.h +++ b/include/pernix/fallback/scalar_decompression.h @@ -79,6 +79,56 @@ __always_inline auto unpack_epi32_fallback(const u8* __restrict__ input, return unpack_epi32_fallback_inner(input, 0, elements); } } + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v && + std::is_floating_point_v) +__always_inline void unpack_and_dequantize_fallback_inner(const u8* __restrict__ input, const u8 bit_offset, + const u32 elements, const ScaleType scale, + ScaleType* __restrict__ output) { + constexpr u32 bits_in_type = sizeof(T) * 8; + constexpr u32 bitmask = BIT_WIDTH == bits_in_type + ? std::numeric_limits::max() + : (1U << BIT_WIDTH) - 1U; + + std::size_t idx = 0; + u8 bits_in_buffer = 8 - bit_offset; + u64 buffer = static_cast(input[idx++]) >> bit_offset; + +#pragma GCC unroll 64 + for (u32 i = 0; i < elements; i++) { + while (BIT_WIDTH > bits_in_buffer) { + const auto next_value = static_cast(input[idx++]) << bits_in_buffer; + buffer |= next_value; + bits_in_buffer += 8; + } + + const u32 raw_value = static_cast(buffer & bitmask); + i32 unpacked = 0; + if constexpr (SIGN_VALUES) { + unpacked = sign_extend(raw_value); + } else { + unpacked = static_cast(raw_value); + } + output[i] = static_cast(unpacked) * scale; + + buffer >>= BIT_WIDTH; + bits_in_buffer -= BIT_WIDTH; + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +__always_inline void unpack_and_dequantize_fallback(const u8* __restrict__ input, const u32 elements, + const ScaleType scale, ScaleType* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); + } else { + return unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); + } +} } template @@ -90,14 +140,7 @@ int decompress_block_fallback(const void* __restrict__ input_ptr, const f32 scal constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const std::vector block_values = internal::unpack_epi32_fallback( - input, elements_per_block); - -#pragma GCC unroll 512 - for (u32 i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi32(block_values[i], scale); - } - + internal::unpack_and_dequantize_fallback(input, elements_per_block, scale, output); return 0; } @@ -110,14 +153,7 @@ int decompress_block_fallback(const void* __restrict__ input_ptr, const f64 scal constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const std::vector block_values = internal::unpack_epi32_fallback( - input, elements_per_block); - -#pragma GCC unroll 512 - for (u32 i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi64(block_values[i], scale); - } - + internal::unpack_and_dequantize_fallback(input, elements_per_block, scale, output); return 0; } diff --git a/include/pernix/x86/avx2/avx2_compression.h b/include/pernix/x86/avx2/avx2_compression.h index d2670e2..e7e3e9a 100644 --- a/include/pernix/x86/avx2/avx2_compression.h +++ b/include/pernix/x86/avx2/avx2_compression.h @@ -136,10 +136,10 @@ __always_inline static __m256i mm256_blend_epi8(const __m256i X, const __m256i Y * @brief Emulate per-byte left shifts on 128-bit vectors. */ __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0xff00); + const __m128i mask = _mm_set1_epi16(static_cast(0xff00u)); const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); - return mm_blend_epi8(low_half, high_half, 0xaa); + return mm_blend_epi8(low_half, high_half, static_cast(0xaau)); } /** @@ -149,17 +149,17 @@ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i coun const __m128i mask = _mm_set1_epi16(0x00ff); const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); - return mm_blend_epi8(low_half, high_half, 0xaa); + return mm_blend_epi8(low_half, high_half, static_cast(0xaau)); } /** * @brief Emulate per-byte left shifts on 256-bit vectors. */ __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0xff00); + const __m256i mask = _mm256_set1_epi16(static_cast(0xff00u)); const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); - return mm256_blend_epi8(low_half, high_half, 0xaa); + return mm256_blend_epi8(low_half, high_half, static_cast(0xaau)); } /** @@ -169,7 +169,7 @@ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i c const __m256i mask = _mm256_set1_epi16(0x00ff); const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); - return mm256_blend_epi8(low_half, high_half, 0xaa); + return mm256_blend_epi8(low_half, high_half, static_cast(0xaau)); } #endif @@ -310,7 +310,7 @@ __always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i const __m256i packed8 = _mm256_packus_epi16(permuted, zero); const __m256i even = _mm256_and_si256(packed8, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(packed8, _mm256_set1_epi16(0xFF00)); + const __m256i odd = _mm256_and_si256(packed8, _mm256_set1_epi16(static_cast(0xFF00u))); const __m256i pair16 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); const __m256i extended = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(pair16)); diff --git a/include/pernix/x86/avx2/avx2_tables.h b/include/pernix/x86/avx2/avx2_tables.h index f62250f..20f9cf4 100644 --- a/include/pernix/x86/avx2/avx2_tables.h +++ b/include/pernix/x86/avx2/avx2_tables.h @@ -749,7 +749,7 @@ struct unpack_tables_avx2 { std::array shifts{}; for (std::size_t lane = 0; lane < 8; ++lane) { - const int bit_offset = lane * BIT_WIDTH; + const int bit_offset = static_cast(lane * BIT_WIDTH); const int bit_in_byte = bit_offset % 8; const int left_shift = 32 - BIT_WIDTH - bit_in_byte; shifts[lane] = left_shift; diff --git a/include/pernix/x86/avx512vbmi/packing.h b/include/pernix/x86/avx512vbmi/packing.h index 301afb6..ebdd066 100644 --- a/include/pernix/x86/avx512vbmi/packing.h +++ b/include/pernix/x86/avx512vbmi/packing.h @@ -31,9 +31,12 @@ __always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { } else { const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - const __m128i permuted1 = _mm_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m128i permuted2 = _mm_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m128i permuted3 = _mm_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); + const __m128i permuted1 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask1), + tables::get_permute1(), masked); + const __m128i permuted2 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask2), + tables::get_permute2(), masked); + const __m128i permuted3 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask3), + tables::get_permute3(), masked); const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); @@ -68,7 +71,7 @@ __always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { return _mm_cvtepi32_epi8(combined2); } else if constexpr (BIT_WIDTH == 3) { const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); + const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(static_cast(0xFF00u))); const __m128i pair6 = _mm_or_si128(even, _mm_srli_epi16(odd, 5)); const __m128i packed12 = _mm_or_si128(pair6, _mm_srli_epi32(pair6, 10)); @@ -81,7 +84,7 @@ __always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { return _mm_cvtepi16_epi8(combined); } else { const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); + const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(static_cast(0xFF00u))); const __m128i shifted = _mm_or_si128(even, _mm_srli_epi16(odd, 8 - BIT_WIDTH)); return mm_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); @@ -137,9 +140,12 @@ __always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) } else { const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); + const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask1), + tables::get_permute1(), masked); + const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask2), + tables::get_permute2(), masked); + const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask3), + tables::get_permute3(), masked); const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); @@ -175,7 +181,7 @@ __always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { return _mm256_castsi128_si256(_mm256_cvtepi32_epi8(combined2)); } else if constexpr (BIT_WIDTH == 3) { const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); + const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(static_cast(0xFF00u))); const __m256i pair6 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 5)); const __m256i packed12 = _mm256_or_si256(pair6, _mm256_srli_epi32(pair6, 10)); @@ -189,7 +195,7 @@ __always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { return _mm256_castsi128_si256(_mm256_cvtepi16_epi8(combined)); } else { const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); + const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(static_cast(0xFF00u))); const __m256i shifted = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); return mm256_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); @@ -245,9 +251,12 @@ __always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) } else { const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); + const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask1), + tables::get_permute1(), masked); + const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask2), + tables::get_permute2(), masked); + const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask3), + tables::get_permute3(), masked); const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); @@ -283,7 +292,7 @@ __always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { return _mm512_castsi128_si512(_mm512_cvtepi32_epi8(combined2)); } else if constexpr (BIT_WIDTH == 3) { const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); + const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(static_cast(0xFF00u))); const __m512i pair6 = _mm512_or_si512(even, _mm512_srli_epi16(odd, 5)); const __m512i packed12 = _mm512_or_si512(pair6, _mm512_srli_epi32(pair6, 10)); @@ -297,7 +306,7 @@ __always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { return _mm512_castsi256_si512(_mm512_cvtepi16_epi8(combined)); } else { const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); + const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(static_cast(0xFF00u))); const __m512i shifted = _mm512_or_si512(even, _mm512_srli_epi16(odd, 8 - BIT_WIDTH)); return mm512_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); diff --git a/include/pernix/x86/avx512vbmi/unpacking.h b/include/pernix/x86/avx512vbmi/unpacking.h index 647d3ad..e021819 100644 --- a/include/pernix/x86/avx512vbmi/unpacking.h +++ b/include/pernix/x86/avx512vbmi/unpacking.h @@ -17,7 +17,7 @@ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i coun } __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0xff00); + const __m128i mask = _mm_set1_epi16(static_cast(0xff00u)); const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); @@ -29,7 +29,7 @@ __always_inline static __m128i _mm_slli_epi8(const __m128i a, const i8 imm8) { __always_inline static __m128i _mm_srli_epi8(const __m128i a, const int imm8) { const __m128i lo_mask = _mm_set1_epi16(0x00ff); - const __m128i hi_mask = _mm_set1_epi16(0xff00); + const __m128i hi_mask = _mm_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m128i lo = _mm_srl_epi16(_mm_and_si128(a, lo_mask), shift); @@ -40,7 +40,7 @@ __always_inline static __m128i _mm_srli_epi8(const __m128i a, const int imm8) { __always_inline static __m128i _mm_srai_epi8(const __m128i a, const i8 imm8) { const __m128i lo_mask = _mm_set1_epi16(0x00ff); - const __m128i hi_mask = _mm_set1_epi16(0xff00); + const __m128i hi_mask = _mm_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m128i hi = _mm_and_si128(_mm_sra_epi16(a, shift), hi_mask); @@ -187,7 +187,7 @@ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i c } __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0xff00); + const __m256i mask = _mm256_set1_epi16(static_cast(0xff00u)); const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); @@ -199,7 +199,7 @@ __always_inline static __m256i _mm256_slli_epi8(const __m256i a, const i8 imm8) __always_inline static __m256i _mm256_srli_epi8(const __m256i a, const i8 imm8) { const __m256i lo_mask = _mm256_set1_epi16(0x00ff); - const __m256i hi_mask = _mm256_set1_epi16(0xff00); + const __m256i hi_mask = _mm256_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m256i lo = _mm256_srl_epi16(_mm256_and_si256(a, lo_mask), shift); @@ -210,7 +210,7 @@ __always_inline static __m256i _mm256_srli_epi8(const __m256i a, const i8 imm8) __always_inline static __m256i _mm256_srai_epi8(const __m256i a, const i8 imm8) { const __m256i lo_mask = _mm256_set1_epi16(0x00ff); - const __m256i hi_mask = _mm256_set1_epi16(0xff00); + const __m256i hi_mask = _mm256_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m256i hi = _mm256_and_si256(_mm256_sra_epi16(a, shift), hi_mask); @@ -357,7 +357,7 @@ __always_inline static __m512i _mm512_srlv_epi8(const __m512i a, const __m512i c } __always_inline static __m512i _mm512_sllv_epi8(const __m512i a, const __m512i count) { - const __m512i mask = _mm512_set1_epi16(0xff00); + const __m512i mask = _mm512_set1_epi16(static_cast(0xff00u)); const __m512i low_half = _mm512_sllv_epi16(a, _mm512_andnot_si512(mask, count)); const __m512i high_half = _mm512_sllv_epi16(_mm512_and_si512(mask, a), _mm512_srli_epi16(count, 8)); return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); @@ -369,7 +369,7 @@ __always_inline static __m512i _mm512_slli_epi8(const __m512i a, const i8 imm8) __always_inline static __m512i _mm512_srli_epi8(const __m512i a, const i8 imm8) { const __m512i lo_mask = _mm512_set1_epi16(0x00ff); - const __m512i hi_mask = _mm512_set1_epi16(0xff00); + const __m512i hi_mask = _mm512_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m512i lo = _mm512_srl_epi16(_mm512_and_si512(a, lo_mask), shift); @@ -380,7 +380,7 @@ __always_inline static __m512i _mm512_srli_epi8(const __m512i a, const i8 imm8) __always_inline static __m512i _mm512_srai_epi8(const __m512i a, const i8 imm8) { const __m512i lo_mask = _mm512_set1_epi16(0x00ff); - const __m512i hi_mask = _mm512_set1_epi16(0xff00); + const __m512i hi_mask = _mm512_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m512i hi = _mm512_and_si512(_mm512_sra_epi16(a, shift), hi_mask); diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp index 54dd0a6..44eedd6 100644 --- a/tests/fallback_tests.cpp +++ b/tests/fallback_tests.cpp @@ -475,6 +475,56 @@ TEST(FallbackEdgeTest, SignValuesFalseTreatsPackedValuesAsUnsigned) { EXPECT_EQ(unsigned_output[0], 15.0f); } +TEST(FallbackEdgeTest, FixedBlockInvariantAndMixedSignMultiBlockRoundTrip) { + constexpr u32 BS = 64; + constexpr u32 BW = 12; + constexpr u32 blocks = 3; + constexpr u32 EPB = (BS * 8) / BW; + constexpr u32 total = EPB * blocks; + + static_assert(BS == 64); + static_assert(EPB == 42); + EXPECT_EQ(pernix_compressed_block_size(), BS); + EXPECT_EQ(pernix_elements_per_block(BW), EPB); + + std::array input_f32{}; + std::array input_f64{}; + for (u32 i = 0; i < total; ++i) { + const auto centered = static_cast(i % 31U) - 15; + input_f32[i] = static_cast(centered) * 0.25f; + input_f64[i] = static_cast(centered) * 0.25; + } + + std::array compressed_f32{}; + std::array compressed_f64{}; + std::array restored_f32{}; + std::array restored_f64{}; + + auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input_f32.data(), 4.0f, + compressed_f32.data(), blocks); + ASSERT_EQ(status, PERNIX_STATUS_OK); + status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed_f32.data(), 0.25f, + restored_f32.data(), blocks, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input_f64.data(), 4.0, + compressed_f64.data(), blocks); + ASSERT_EQ(status, PERNIX_STATUS_OK); + status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed_f64.data(), 0.25, + restored_f64.data(), blocks, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + for (u32 i = 0; i < total; ++i) { + EXPECT_FLOAT_EQ(restored_f32[i], input_f32[i]) << "f32 element " << i; + EXPECT_DOUBLE_EQ(restored_f64[i], input_f64[i]) << "f64 element " << i; + } + + std::array unsigned_output{}; + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, compressed_f32.data(), 1.0f, + unsigned_output.data(), false); + ASSERT_EQ(status, PERNIX_STATUS_OK); +} + TEST(FallbackEdgeTest, CApiRejectsInvalidScale) { constexpr u32 BS = 64; constexpr u32 BW = 8; diff --git a/tests/header_only_tests.cpp b/tests/header_only_tests.cpp index 3a229b2..ca58e16 100644 --- a/tests/header_only_tests.cpp +++ b/tests/header_only_tests.cpp @@ -81,12 +81,11 @@ TEST(HeaderOnlyPernix, ScaleHelpersNameCompressionAndDecompressionConventions) { EXPECT_EQ(pernix::compression_scale_from_bmax(1.0f, 0, compression_scale), PERNIX_STATUS_INVALID_ARGUMENT); } -TEST(HeaderOnlyPernix, StatusAliasAndStringsArePublic) { +TEST(HeaderOnlyPernix, StatusStringsArePublic) { pernix::Status status = PERNIX_STATUS_OK; EXPECT_EQ(status, PERNIX_STATUS_OK); EXPECT_STREQ(pernix::status_string(status), "PERNIX_STATUS_OK"); - EXPECT_EQ(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTAION, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); - EXPECT_STREQ(pernix::status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTAION), + EXPECT_STREQ(pernix::status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION), "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION"); EXPECT_STREQ(pernix::status_string(static_cast(123)), "PERNIX_STATUS_UNKNOWN"); } From 13c6a31618e1318836d3444374885ec903a34b80 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Wed, 8 Jul 2026 22:38:45 +0200 Subject: [PATCH 24/43] feat: enhance CMake configuration for fallback stdpar and install consumer tests --- CMakeLists.txt | 56 ++++++++++++++++++++++++------------- README.md | 12 ++++++-- cmake/pernix.pc.in | 9 +++--- cmake/pernixConfig.cmake.in | 6 ++++ src/CMakeLists.txt | 34 +++++++++++++++++----- tests/CMakeLists.txt | 14 ++++++++++ 6 files changed, 99 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b9aea32..982278f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,27 +1,45 @@ cmake_minimum_required(VERSION 3.24) project(pernix VERSION 0.1.0 LANGUAGES CXX C) -option(PERNIX_ENABLE_TESTS "Enable tests for pernix" on) -option(PERNIX_ENABLE_CODE_COVERAGE "Enable code coverage for pernix (requires lcov/genhtml)" off) -option(PERNIX_ENABLE_LTO "Enable Link Time Optimization (LTO)" off) -option(PERNIX_ENABLE_INSTALL "Enable installation of pernix" on) -option(PERNIX_ENABLE_DOXYGEN "Build documentation with Doxygen." off) -option(PERNIX_INSTALL_DOCS "Install documentation with pernix" off) -option(PERNIX_ENABLE_EXAMPLES "Build pernix example programs" off) - -option(PERNIX_ENABLE_X86_BMI2 "Build x86 BMI2 backend" ON) -option(PERNIX_ENABLE_X86_AVX2 "Build x86 AVX2 backend" ON) -option(PERNIX_ENABLE_X86_AVX512VBMI "Build x86 AVX512-VBMI backend" ON) -option(PERNIX_ENABLE_ARM64_NEON "Build arm64 NEON backend" ON) -option(PERNIX_ENABLE_ARM64_SVE2 "Build arm64 SVE2 backend" ON) -option(PERNIX_ENABLE_FALLBACK_STDPAR "Build fallback stdpar backend" ON) -option(PERNIX_ENABLE_FALLBACK_SIMD "Build fallback simd skeleton backend" OFF) - -option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" off) - -option(PERNIX_ENABLE_FORTRAN_BINDINGS "Build Fortran bindings for pernix" off) +option(BUILD_SHARED_LIBS "Build pernix as a shared library by default" ON) +option(PERNIX_ENABLE_TESTS "Build pernix unit and integration tests" ON) +option(PERNIX_ENABLE_CODE_COVERAGE "Enable code coverage for pernix tests (requires lcov/genhtml)" OFF) +option(PERNIX_ENABLE_LTO "Enable interprocedural optimization/link-time optimization" OFF) +option(PERNIX_ENABLE_INSTALL "Install pernix library, headers, and package metadata" ON) +option(PERNIX_ENABLE_DOXYGEN "Build API documentation with Doxygen" OFF) +option(PERNIX_INSTALL_DOCS "Install generated Doxygen documentation; requires PERNIX_ENABLE_DOXYGEN=ON" OFF) +option(PERNIX_ENABLE_EXAMPLES "Build pernix example programs" OFF) +option(PERNIX_ENABLE_INSTALL_CONSUMER_TESTS "Add CTest checks for consuming an installed pernix package" ON) + +option(PERNIX_ENABLE_X86_BMI2 "Build x86 BMI2 backend when targeting x86" ON) +option(PERNIX_ENABLE_X86_AVX2 "Build x86 AVX2 backend when targeting x86" ON) +option(PERNIX_ENABLE_X86_AVX512VBMI "Build x86 AVX512-VBMI backend when targeting x86" ON) +option(PERNIX_ENABLE_ARM64_NEON "Build arm64 NEON backend when targeting arm64" ON) +option(PERNIX_ENABLE_ARM64_SVE2 "Build arm64 SVE2 backend when targeting arm64" ON) +set(PERNIX_ENABLE_FALLBACK_STDPAR "OFF" CACHE STRING "Enable the header-only stdpar fallback backend: OFF, AUTO, or ON. AUTO enables it only when TBB is found.") +set_property(CACHE PERNIX_ENABLE_FALLBACK_STDPAR PROPERTY STRINGS AUTO ON OFF) +option(PERNIX_ENABLE_FALLBACK_SIMD "Build experimental fallback SIMD skeleton backend" OFF) + +option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" OFF) + +option(PERNIX_ENABLE_FORTRAN_BINDINGS "Build optional Fortran bindings and round-trip test" OFF) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") +set(PERNIX_BUNDLE_SIMDE_FOR_INSTALL OFF) + +string(TOUPPER "${PERNIX_ENABLE_FALLBACK_STDPAR}" PERNIX_ENABLE_FALLBACK_STDPAR) +set(PERNIX_ENABLE_FALLBACK_STDPAR "${PERNIX_ENABLE_FALLBACK_STDPAR}" CACHE STRING "Enable the header-only stdpar fallback backend: OFF, AUTO, or ON. AUTO enables it only when TBB is found." FORCE) +if (NOT PERNIX_ENABLE_FALLBACK_STDPAR MATCHES "^(AUTO|ON|OFF)$") + message(FATAL_ERROR "PERNIX_ENABLE_FALLBACK_STDPAR must be AUTO, ON, or OFF") +endif () + +if (PERNIX_INSTALL_DOCS AND NOT PERNIX_ENABLE_DOXYGEN) + message(FATAL_ERROR "PERNIX_INSTALL_DOCS requires PERNIX_ENABLE_DOXYGEN=ON") +endif () + +if (PERNIX_ENABLE_INSTALL_CONSUMER_TESTS AND PERNIX_ENABLE_TESTS AND NOT PERNIX_ENABLE_INSTALL) + message(FATAL_ERROR "PERNIX_ENABLE_INSTALL_CONSUMER_TESTS requires PERNIX_ENABLE_INSTALL=ON") +endif () #[===[ set(PERNIX_BUNDLE_SIMDE_FOR_INSTALL OFF) diff --git a/README.md b/README.md index c6d0a4f..d71c228 100644 --- a/README.md +++ b/README.md @@ -102,14 +102,22 @@ cmake -E chdir build cmake -DCMAKE_BUILD_TYPE=Release -DPERNIX_ENABLE_TESTS=off cmake --build build --config Release ``` -The shared library is written under `build/src`. +The library is written under `build/src`. Shared builds are the default; pass `-DBUILD_SHARED_LIBS=OFF` for a static +library. Optional build flags: * `-DPERNIX_ENABLE_TESTS=ON` builds the test suite. * `-DPERNIX_ENABLE_EXAMPLES=ON` builds the C and C++ examples. * `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON` builds the Fortran bindings and Fortran round-trip program. -* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF` disables the stdpar fallback backend and its TBB dependency. +* `-DPERNIX_ENABLE_INSTALL=ON` installs the library, headers, pkg-config file, and CMake package files. +* `-DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON` adds a CTest that installs PERNIX into a temporary prefix and verifies + C and C++ consumers with `find_package(pernix CONFIG REQUIRED)`. +* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF` is the default and avoids the TBB dependency. Use `AUTO` to enable the + header-only stdpar fallback backend only when TBB is found, or `ON` to require TBB and fail if missing. The compiled + library target does not currently export stdpar dispatch. +* `-DPERNIX_ENABLE_X86_AVX2=OFF`, `-DPERNIX_ENABLE_X86_BMI2=OFF`, and + `-DPERNIX_ENABLE_X86_AVX512VBMI=OFF` disable individual x86 SIMD backends. ## Usage Examples diff --git a/cmake/pernix.pc.in b/cmake/pernix.pc.in index 87fdb27..00f60cd 100644 --- a/cmake/pernix.pc.in +++ b/cmake/pernix.pc.in @@ -1,11 +1,12 @@ -prefix=@CMAKE_INSTALL_PREFIX@ +prefix=${pcfiledir}/../.. exec_prefix=${prefix} -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ +libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ name: Pernix description: A high-performance C++ library for de/compression of floating-point data version: @NORMALIZED_VERSION@ +Requires: @PERNIX_PC_REQUIRES@ Libs: -L${libdir} -lpernix -Cflags: -I${includedir} \ No newline at end of file +Cflags: -I${includedir} diff --git a/cmake/pernixConfig.cmake.in b/cmake/pernixConfig.cmake.in index fe3eb88..6819581 100644 --- a/cmake/pernixConfig.cmake.in +++ b/cmake/pernixConfig.cmake.in @@ -1,5 +1,7 @@ @PACKAGE_INIT@ +include(CMakeFindDependencyMacro) + if (@PERNIX_USE_SIMDE@) find_package(simde CONFIG QUIET) if (NOT TARGET simde::simde AND @PERNIX_BUNDLE_SIMDE_FOR_INSTALL@) @@ -13,6 +15,10 @@ if (@PERNIX_USE_SIMDE@) endif () endif () +if (@PERNIX_BUILD_FALLBACK_STDPAR@) + find_dependency(TBB) +endif () + include("${CMAKE_CURRENT_LIST_DIR}/pernixTargets.cmake") check_required_components(pernix) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f6f449a..74ab4b8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,8 @@ include(GNUInstallDirs) include(CMakePackageConfigHelpers) -add_library(pernix SHARED) +add_library(pernix) +add_library(pernix::pernix ALIAS pernix) add_library(pernix_header_only INTERFACE) add_library(pernix::pernix_header_only ALIAS pernix_header_only) @@ -44,6 +45,9 @@ target_include_directories(pernix_header_only ) target_compile_definitions(pernix PRIVATE PERNIX_BUILD_LIB=1) +if (BUILD_SHARED_LIBS) + target_compile_definitions(pernix PUBLIC PERNIX_SHARED=1) +endif () function(pernix_set_source_options) set(options) @@ -66,14 +70,26 @@ if (PERNIX_USE_SIMDE) target_compile_definitions(pernix_header_only INTERFACE PERNIX_USE_SIMDE=1) endif () -if (PERNIX_ENABLE_FALLBACK_STDPAR) - find_package(TBB REQUIRED) - target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_STDPAR=1) - target_link_libraries(pernix PUBLIC TBB::tbb) - target_link_libraries(pernix_header_only INTERFACE TBB::tbb) +set(PERNIX_BUILD_FALLBACK_STDPAR OFF) +if (NOT PERNIX_ENABLE_FALLBACK_STDPAR STREQUAL "OFF") + if (PERNIX_ENABLE_FALLBACK_STDPAR STREQUAL "ON") + find_package(TBB REQUIRED) + else () + find_package(TBB QUIET) + endif () + + if (TARGET TBB::tbb) + set(PERNIX_BUILD_FALLBACK_STDPAR ON) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_STDPAR=1) + target_link_libraries(pernix_header_only INTERFACE TBB::tbb) + message(STATUS "PERNIX header-only fallback stdpar backend enabled (TBB found)") + elseif (PERNIX_ENABLE_FALLBACK_STDPAR STREQUAL "AUTO") + message(STATUS "PERNIX header-only fallback stdpar backend disabled (TBB not found; set PERNIX_ENABLE_FALLBACK_STDPAR=ON to require it)") + endif () endif () if (PERNIX_ENABLE_FALLBACK_SIMD) + target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_SIMD=1) target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_SIMD=1) endif () @@ -173,6 +189,8 @@ if (PERNIX_DISABLE_AVX512) target_compile_definitions(pernix_header_only INTERFACE PERNIX_DISABLE_AVX512=1) endif () +set(PERNIX_PC_REQUIRES "") + configure_file("${PROJECT_SOURCE_DIR}/cmake/pernix.pc.in" "${PROJECT_BINARY_DIR}/pernix.pc" @ONLY) if (PERNIX_ENABLE_INSTALL) @@ -185,7 +203,9 @@ if (PERNIX_ENABLE_INSTALL) install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/pernix" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - FILES_MATCHING PATTERN "*.*h" + FILES_MATCHING + PATTERN "*.h" + PATTERN "*.hpp" ) if (PERNIX_USE_SIMDE AND PERNIX_BUNDLE_SIMDE_FOR_INSTALL) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 18f7271..c18c3ef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -41,6 +41,20 @@ target_link_libraries(pernix_c_api_validation PRIVATE pernix) target_include_directories(pernix_c_api_validation PRIVATE ${PROJECT_SOURCE_DIR}/include) add_test(NAME pernix_c_api_validation COMMAND pernix_c_api_validation) +if (PERNIX_ENABLE_INSTALL_CONSUMER_TESTS) + add_test(NAME pernix_install_consumer + COMMAND ${CMAKE_COMMAND} + -DPERNIX_BUILD_DIR=${PROJECT_BINARY_DIR} + -DPERNIX_CONSUMER_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/install_consumer + -DPERNIX_CONSUMER_BUILD_DIR=${PROJECT_BINARY_DIR}/install-consumer-build + -DPERNIX_INSTALL_PREFIX=${PROJECT_BINARY_DIR}/install-consumer-prefix + -DPERNIX_TEST_CONFIG=$ + -DPERNIX_C_COMPILER=${CMAKE_C_COMPILER} + -DPERNIX_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -P ${CMAKE_CURRENT_SOURCE_DIR}/install_consumer/run.cmake + ) +endif () + include(GoogleTest) foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) gtest_discover_tests(pernix_tests_${BLOCK_SIZE} From de84152a4b747903183619d1e2598021921df8dc Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Wed, 8 Jul 2026 22:56:56 +0200 Subject: [PATCH 25/43] docs: update README and examples for clarity and consistency in API usage --- README.md | 282 +++++++++++++++++++++------------ examples/c/roundtrip.c | 13 +- examples/cpp/roundtrip.cpp | 4 +- examples/fortran/roundtrip.f90 | 8 +- 4 files changed, 200 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index d71c228..434a27a 100644 --- a/README.md +++ b/README.md @@ -1,127 +1,132 @@ -# PERNIX: Floating-Point Number De/Compression on CPUs +# PERNIX -PERNIX is a high-throughput floating-point compression library for CPU-based scientific workloads. It quantizes -floating-point values to a configurable bit width and packs them into fixed-size compressed blocks, reducing memory and -communication bandwidth while keeping decompression fast. +PERNIX is a small C/C++ library for CPU-based scientific data compression. It quantizes `float` or `double` values to a +user-chosen bit width and packs the quantized integers into fixed 64-byte blocks. The main goal is fast block +decompression with a portable fallback path and optional SIMD backends. -The library provides: +The public API includes: -* C++ API functions such as `pernix::compress_block` and `pernix::decompress_block` -* C ABI wrappers named `pernix_compress_block_f32`, `pernix_decompress_block_f32`, and matching `_f64` variants -* Optional Fortran bindings in `bindings/fortran` -* SIMD-optimized backends where available, with a portable fallback backend +* C++ functions in ``, including `pernix::compress_block` and `pernix::decompress_block` +* a plain C ABI in ``, including `pernix_compress_block_f32` and `pernix_decompress_block_f32` +* `_f64` variants for double-precision input/output +* optional Fortran bindings that call the C ABI -## Quantization Convention +## Block Format -For a block of floating-point numbers `x_i` and a bit width `N`, PERNIX uses the block maximum magnitude: +The normal PERNIX block is exactly 64 bytes, or 512 bits. For bit width `N`, one block stores: ```text -bmax = max(abs(x_i)) -scale = bmax / (2^(N - 1) - 1) +elements_per_block = (64 * 8) / N ``` -The current compression API expects the inverse scale, because compression computes: +Valid public bit widths are `1..24`. Public helpers expose these constants and calculations: -```text -quantized = round(input * scale_inverse) -``` +* `pernix::compressed_block_size()` / `pernix_compressed_block_size()` returns `64` +* `pernix::elements_per_block(bit_width)` / `pernix_elements_per_block(bit_width)` returns the 64-byte element count +* `pernix::is_valid_bit_width(bit_width)` / `pernix_is_valid_bit_width(bit_width)` checks `1..24` +* `pernix::min_bit_width()` and `pernix::max_bit_width()` return `1` and `24` + +The implementation also accepts explicit block sizes `128`, `256`, `512`, and `1024` for internal/test coverage. The +documented interchange format is the 64-byte block. + +Packed values are written as a byte stream, least-significant bits first within each value and byte. Compression +zero-fills the output block before packing, so unused tail bits in a block are zero. When `sign_values=true`, decompression +sign-extends each `N`-bit value before multiplying by scale. When `sign_values=false`, decompression treats the packed +value as unsigned. For `bit_width == 1`, signed fallback compression clamps to binary `0/1`. + +The scalar fallback packs and unpacks byte-by-byte, so the serialized 64-byte block is not a native integer dump. Backends +are expected to produce compatible blocks for the same inputs, bit width, scale, and sign mode. + +## Quantization And Scale -Decompression expects the forward scale, because decompression computes: +For a block with maximum magnitude `bmax` and bit width `N`, PERNIX uses this forward decompression scale: ```text -output = quantized * scale +scale = bmax / (2^(N - 1) - 1) ``` -In practice, compute the decompression scale from `bmax`, pass its inverse to `compress_block`, and pass the forward -scale to `decompress_block`. The clearer helper names are: +For `N == 1`, the denominator is treated as `1`. For `bmax == 0`, helper functions return a small positive scale rather +than zero. -* `pernix_decompression_scale_f32` / `pernix_decompression_scale_f64` -* `pernix_compression_scale_f32` / `pernix_compression_scale_f64` -* `pernix_inverse_scale_f32` / `pernix_inverse_scale_f64` -* `pernix::decompression_scale_from_bmax`, `pernix::compression_scale_from_bmax`, and `pernix::inverse_scale` +The current compression API expects the inverse scale: -The older `pernix_scale_f32`, `pernix_scale_f64`, and `pernix::scale_from_bmax` names are kept as compatibility aliases -for the forward decompression scale. Scale-from-`bmax` helpers return a small positive scale for `bmax == 0`. +```text +quantized = round(input * inverse_scale) +``` -Scale arguments passed to compression and decompression must be finite and greater than zero. Passing zero, negative, -NaN, or infinity returns `PERNIX_STATUS_INVALID_ARGUMENT`. +The decompression API expects the forward scale: -## Block Format and Limits +```text +output = quantized * scale +``` -PERNIX uses 64-byte (512-bit) compressed blocks by default. For bit width `N`, one 64-byte block stores -`(64 * 8) / N` values. The fallback implementation currently supports bit widths `1..24`. Public APIs also accept an -explicit `block_size`; supported values are `64`, `128`, `256`, `512`, and `1024`, but the documented default and normal -format is a fixed 64-byte block. +Use these helpers to avoid mixing the two conventions: -Packed values are signed by default during decompression. A `sign_values` argument is available in the C ABI and in the -C++ API to request unsigned interpretation. +* C++: `pernix::decompression_scale_from_bmax`, `pernix::compression_scale_from_bmax`, `pernix::inverse_scale` +* C: `pernix_decompression_scale_f32`, `pernix_compression_scale_f32`, `pernix_inverse_scale_f32` +* C f64: the same names with `_f64` -The public helper APIs expose the fixed-format constants: +Compatibility aliases `pernix::scale_from_bmax`, `pernix_scale_f32`, and `pernix_scale_f64` compute the forward +decompression scale. -* `pernix_min_bit_width()` / `pernix::min_bit_width()` return `1` -* `pernix_max_bit_width()` / `pernix::max_bit_width()` return `24` -* `pernix_compressed_block_size()` / `pernix::compressed_block_size()` return `64` -* `pernix_elements_per_block(bit_width)` / `pernix::elements_per_block(bit_width)` return `(64 * 8) / bit_width`, or - `0` for an invalid bit width -* `pernix_is_valid_bit_width(bit_width)` / `pernix::is_valid_bit_width(bit_width)` validate the public `1..24` range -* `pernix_is_valid_block_size(block_size)` / `pernix::is_valid_block_size(block_size)` validate the currently accepted - block sizes +Scale arguments passed to compression and decompression must be finite and greater than zero. Zero, negative, NaN, or +infinite scales return `PERNIX_STATUS_INVALID_ARGUMENT`. -Input values should be finite for portable behavior. The scalar fallback clamps non-finite or out-of-range values before -narrowing, but that behavior is not a cross-backend contract. +## API Contracts -## Status Codes +All public compression and decompression calls return `pernix_status` (`pernix::Status` in C++). -The C ABI returns `pernix_status`: +Status values: * `PERNIX_STATUS_OK`: success -* `PERNIX_STATUS_INVALID_ARGUMENT`: null pointer or invalid parameter -* `PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH`: unsupported bit width -* `PERNIX_STATUS_UNSUPPORTED_BACKEND`: unknown backend value -* `PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE`: unsupported block size -* `PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION`: backend was requested but is not available for this build or CPU +* `PERNIX_STATUS_INVALID_ARGUMENT`: null pointer, invalid span size, zero block count, or invalid scale +* `PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH`: bit width outside `1..24` +* `PERNIX_STATUS_UNSUPPORTED_BACKEND`: backend enum value is unknown +* `PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE`: block size is not supported +* `PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION`: backend is not compiled in or is unavailable on this CPU -Use `pernix_status_string(status)` or `pernix::status_string(status)` for diagnostic text. +Use `pernix::status_string(status)` or `pernix_status_string(status)` for readable names. -## f32 and f64 APIs +For one block: -Use the `_f32` functions for `float` input/output and `_f64` functions for `double` input/output. Both variants pack to -the same integer bit stream for the selected bit width. The scale type matches the floating-point type: -`float` for `_f32`, `double` for `_f64`. +* compression reads at least `(block_size * 8) / bit_width` `float` or `double` values +* compression writes exactly `block_size` bytes +* decompression reads exactly `block_size` bytes +* decompression writes at least `(block_size * 8) / bit_width` `float` or `double` values -The C++ wrapper overloads return `pernix::Status`, an alias for the C ABI `pernix_status`. `std::span` arguments are -borrowed only for the duration of the call. For one 64-byte block, compression reads at least -`pernix::elements_per_block(bit_width)` input values and writes `pernix::compressed_block_size()` bytes. Multi-block -calls multiply those counts by `blocks`. +For `*_blocks` calls, multiply those sizes by `blocks`. `blocks == 0` is invalid. The C ABI validates null pointers and +basic parameters, but it cannot validate buffer lengths. The C++ `std::span` wrappers validate span sizes before calling +the lower-level kernels. -## Compiling +The `_f32` APIs operate on `float` data and take `float` scale values. The `_f64` APIs operate on `double` data and take +`double` scale values. Both use the same packed integer block format for a given bit width. -```bash -cmake -E make_directory build -cmake -E chdir build cmake -DCMAKE_BUILD_TYPE=Release -DPERNIX_ENABLE_TESTS=off ../ -cmake --build build --config Release -``` +Given the same backend, inputs, bit width, scale, and sign mode, behavior is deterministic. Different backends are tested +for compatible results, but exact floating-point details should not be treated as a cross-backend ABI guarantee beyond the +documented quantization model. -The library is written under `build/src`. Shared builds are the default; pass `-DBUILD_SHARED_LIBS=OFF` for a static -library. +## Backends -Optional build flags: +`PERNIX_BACKEND_FALLBACK` is the portable scalar backend and is always available. `PERNIX_BACKEND_AUTO` selects an +available optimized backend when one is compiled and supported on the host CPU, otherwise it falls back. -* `-DPERNIX_ENABLE_TESTS=ON` builds the test suite. -* `-DPERNIX_ENABLE_EXAMPLES=ON` builds the C and C++ examples. -* `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON` builds the Fortran bindings and Fortran round-trip program. -* `-DPERNIX_ENABLE_INSTALL=ON` installs the library, headers, pkg-config file, and CMake package files. -* `-DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON` adds a CTest that installs PERNIX into a temporary prefix and verifies - C and C++ consumers with `find_package(pernix CONFIG REQUIRED)`. -* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF` is the default and avoids the TBB dependency. Use `AUTO` to enable the - header-only stdpar fallback backend only when TBB is found, or `ON` to require TBB and fail if missing. The compiled - library target does not currently export stdpar dispatch. -* `-DPERNIX_ENABLE_X86_AVX2=OFF`, `-DPERNIX_ENABLE_X86_BMI2=OFF`, and - `-DPERNIX_ENABLE_X86_AVX512VBMI=OFF` disable individual x86 SIMD backends. +Current backend enum values: -## Usage Examples +* `PERNIX_BACKEND_AUTO` +* `PERNIX_BACKEND_FALLBACK` +* `PERNIX_BACKEND_X86_AVX2` +* `PERNIX_BACKEND_X86_BMI2` +* `PERNIX_BACKEND_X86_AVX512_VBMI` +* `PERNIX_BACKEND_ARM64_NEON` +* `PERNIX_BACKEND_ARM64_SVE` +* `PERNIX_BACKEND_FALLBACK_STDPAR` +* `PERNIX_BACKEND_FALLBACK_SIMD` -### C++ API +x86 SIMD kernels are compiled with per-source ISA flags when enabled. Generic dispatch and fallback code are built for the +baseline target. ARM decompression paths exist, but ARM compression is incomplete/stubbed. The stdpar fallback is currently +a header-only target feature, disabled by default, and the compiled library target does not export stdpar dispatch. + +## C++ Example ```cpp #include @@ -133,8 +138,8 @@ Optional build flags: int main() { constexpr u8 bit_width = 16; - constexpr u32 block_size = 64; - constexpr std::size_t elements = (block_size * 8U) / bit_width; + constexpr u32 block_size = pernix::compressed_block_size(); + constexpr std::size_t elements = pernix::elements_per_block(bit_width); std::array input{}; for (std::size_t i = 0; i < input.size(); ++i) { @@ -145,6 +150,7 @@ int main() { for (float value : input) { bmax = std::max(bmax, std::abs(value)); } + float scale = 0.0f; float inverse_scale = 0.0f; if (pernix::decompression_scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK || @@ -157,17 +163,16 @@ int main() { if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { - return 1; + return 2; } if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, scale, restored) != PERNIX_STATUS_OK) { - return 1; + return 3; } - return 0; } ``` -### C ABI +## C Example ```c #include @@ -194,22 +199,99 @@ int main(void) { if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { - return 1; + return 2; } if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, true) != PERNIX_STATUS_OK) { - return 1; + return 3; } - return 0; } ``` -### Fortran +Complete examples are in `examples/cpp`, `examples/c`, and `examples/fortran`. + +## Fortran + +Fortran bindings are optional. Enable them with: + +```bash +cmake -S . -B build -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON +``` + +The modules in `bindings/fortran/src` bind directly to the C ABI names and currently expose f32 and f64 compression and +decompression entry points. The bindings do not yet install Fortran module files as a packaged Fortran SDK; they are meant +for in-tree builds and examples. + +## Building + +Release build: + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --config Release -j12 +``` + +Common options: + +* `-DBUILD_SHARED_LIBS=OFF`: build a static library instead of the default shared library +* `-DPERNIX_ENABLE_TESTS=ON`: build tests +* `-DPERNIX_ENABLE_EXAMPLES=ON`: build examples +* `-DPERNIX_ENABLE_INSTALL=ON`: install library, headers, CMake package files, and pkg-config metadata +* `-DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON`: add a CTest that installs PERNIX and builds C/C++ consumers +* `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON`: build Fortran bindings and Fortran example/test +* `-DPERNIX_ENABLE_X86_AVX2=OFF`, `-DPERNIX_ENABLE_X86_BMI2=OFF`, + `-DPERNIX_ENABLE_X86_AVX512VBMI=OFF`: disable specific x86 backends +* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF|AUTO|ON`: control the header-only stdpar fallback and TBB dependency + +Install: + +```bash +cmake --install build --prefix /path/to/prefix --config Release +``` + +CMake consumers can use: + +```cmake +find_package(pernix CONFIG REQUIRED) +target_link_libraries(my_target PRIVATE pernix::pernix) +``` + +## Testing + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DPERNIX_ENABLE_TESTS=ON -DPERNIX_ENABLE_EXAMPLES=ON +cmake --build build --config Release -j12 +ctest --test-dir build --output-on-failure +``` + +Examples are built when `PERNIX_ENABLE_EXAMPLES=ON`: + +```bash +./build/examples/pernix_example_cpp +./build/examples/pernix_example_c +``` + +If Fortran bindings are enabled: + +```bash +./build/examples/pernix_example_fortran +``` + +## Limitations + +* Public documentation is centered on fixed 64-byte blocks; larger accepted block sizes are compatibility/internal paths. +* Input values should be finite and within the intended quantization range for portable cross-backend behavior. The scalar + fallback clamps NaN, infinity, and out-of-range scaled values, but that is not yet specified as a cross-backend policy. +* Scale must be positive and finite. +* ARM64 compression backends are incomplete/stubbed. +* `PERNIX_BACKEND_FALLBACK_STDPAR` is not exported by the compiled library target. +* Fortran bindings are buildable in-tree but not yet packaged for installation. +* The packed format is intended to be stable for a given bit width and sign mode, but PERNIX is still pre-1.0. -Fortran bindings call the same C ABI symbols. Enable them with `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON`. A minimal -round-trip program is provided in `bindings/fortran/main.f90`, and a standalone example is in -`examples/fortran/roundtrip.f90`. +## Performance Notes -## Benchmarking +Decompression is the performance-sensitive path. Prefer `PERNIX_BACKEND_AUTO` for normal use so PERNIX can select an +available optimized backend. Use `PERNIX_BACKEND_FALLBACK` when deterministic portable fallback behavior is more important +than backend selection. -A benchmark framework for PERNIX can be found at https://github.com/pc2/pernix-benchmark. +A separate benchmark framework exists at . diff --git a/examples/c/roundtrip.c b/examples/c/roundtrip.c index c4c4acd..ad64824 100644 --- a/examples/c/roundtrip.c +++ b/examples/c/roundtrip.c @@ -7,6 +7,11 @@ int main(void) { u8 compressed[block_size]; float bmax = 0.0f; + if (!pernix_is_valid_bit_width(bit_width) || pernix_compressed_block_size() != block_size || + pernix_elements_per_block(bit_width) != elements) { + return 1; + } + for (int i = 0; i < elements; ++i) { input[i] = ((float)i - 16.0f) * 0.125f; const float magnitude = input[i] < 0.0f ? -input[i] : input[i]; @@ -17,23 +22,23 @@ int main(void) { float inverse_scale = 0.0f; if (pernix_decompression_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK || pernix_inverse_scale_f32(scale, &inverse_scale) != PERNIX_STATUS_OK) { - return 1; + return 2; } if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { - return 1; + return 3; } if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, true) != PERNIX_STATUS_OK) { - return 2; + return 4; } for (int i = 0; i < elements; ++i) { const float diff = restored[i] - input[i]; const float error = diff < 0.0f ? -diff : diff; if (error > scale) { - return 3; + return 5; } } return 0; diff --git a/examples/cpp/roundtrip.cpp b/examples/cpp/roundtrip.cpp index 69153ba..f2381f7 100644 --- a/examples/cpp/roundtrip.cpp +++ b/examples/cpp/roundtrip.cpp @@ -7,8 +7,8 @@ int main() { constexpr u8 bit_width = 16; - constexpr u32 block_size = 64; - constexpr std::size_t elements = (block_size * 8U) / bit_width; + constexpr u32 block_size = pernix::compressed_block_size(); + constexpr std::size_t elements = pernix::elements_per_block(bit_width); std::array input{}; for (std::size_t i = 0; i < input.size(); ++i) { diff --git a/examples/fortran/roundtrip.f90 b/examples/fortran/roundtrip.f90 index 91ac598..033ec1c 100644 --- a/examples/fortran/roundtrip.f90 +++ b/examples/fortran/roundtrip.f90 @@ -12,6 +12,8 @@ program pernix_example_fortran integer(c_int8_t), target :: compressed(block_size) real(c_float) :: bmax real(c_float) :: scale + real(c_float) :: inverse_scale + real(c_float) :: error integer(c_int) :: status integer :: i @@ -21,12 +23,16 @@ program pernix_example_fortran bmax = maxval(abs(input)) scale = bmax / real((2 ** (int(bit_width) - 1)) - 1, c_float) + inverse_scale = 1.0_c_float / scale status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(input), & - 1.0_c_float / scale, c_loc(compressed)) + inverse_scale, c_loc(compressed)) if (status /= PERNIX_STATUS_OK) stop 1 status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(compressed), & scale, c_loc(restored), .true._c_bool) if (status /= PERNIX_STATUS_OK) stop 2 + + error = maxval(abs(restored - input)) + if (error > scale) stop 3 end program pernix_example_fortran From 05c0119590283a18e98acce077ea89adbde3cd00 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 9 Jul 2026 11:47:46 +0200 Subject: [PATCH 26/43] feat: add C API validation and consumer test executables for pernix --- tests/c_api_validation.c | 168 ++++++++++ tests/deterministic_correctness_tests.cpp | 385 ++++++++++++++++++++++ tests/install_consumer/CMakeLists.txt | 14 + tests/install_consumer/main.c | 28 ++ tests/install_consumer/main.cpp | 35 ++ tests/install_consumer/run.cmake | 64 ++++ 6 files changed, 694 insertions(+) create mode 100644 tests/c_api_validation.c create mode 100644 tests/deterministic_correctness_tests.cpp create mode 100644 tests/install_consumer/CMakeLists.txt create mode 100644 tests/install_consumer/main.c create mode 100644 tests/install_consumer/main.cpp create mode 100644 tests/install_consumer/run.cmake diff --git a/tests/c_api_validation.c b/tests/c_api_validation.c new file mode 100644 index 0000000..bc5f313 --- /dev/null +++ b/tests/c_api_validation.c @@ -0,0 +1,168 @@ +#include + +#include +#include + +enum { + bit_width = 8, + block_size = 64, + elements = (block_size * 8) / bit_width +}; + +static int expect_status(pernix_status actual, pernix_status expected) { + return actual == expected ? 0 : 1; +} + +int main(void) { + float input_f32[elements] = {0}; + float output_f32[elements] = {0}; + double input_f64[elements] = {0}; + double output_f64[elements] = {0}; + u8 compressed[block_size] = {0}; + float scale_f32 = 0.0f; + double scale_f64 = 0.0; + + if (expect_status(pernix_scale_f32(1.0f, bit_width, &scale_f32), PERNIX_STATUS_OK) != 0) { + return 1; + } + if (expect_status(pernix_scale_f64(1.0, bit_width, &scale_f64), PERNIX_STATUS_OK) != 0) { + return 2; + } + if (expect_status(pernix_scale_f32(1.0f, bit_width, 0), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 3; + } + if (expect_status(pernix_scale_f32(-1.0f, bit_width, &scale_f32), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 4; + } + if (expect_status(pernix_scale_f64(INFINITY, bit_width, &scale_f64), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 5; + } + + if (!pernix_is_valid_bit_width(pernix_min_bit_width()) || + !pernix_is_valid_bit_width(pernix_max_bit_width()) || + pernix_is_valid_bit_width(0) || + pernix_is_valid_bit_width((u8)(pernix_max_bit_width() + 1U))) { + return 6; + } + if (pernix_compressed_block_size() != block_size || pernix_elements_per_block(bit_width) != elements) { + return 7; + } + if (!pernix_is_valid_block_size(block_size) || pernix_is_valid_block_size(96)) { + return 8; + } + + if (strcmp(pernix_status_string(PERNIX_STATUS_OK), "PERNIX_STATUS_OK") != 0 || + strcmp(pernix_status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION), + "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION") != 0 || + strcmp(pernix_status_string((pernix_status)123), "PERNIX_STATUS_UNKNOWN") != 0) { + return 23; + } + + float decompression_scale_f32 = 0.0f; + float compression_scale_f32 = 0.0f; + float inverse_scale_f32 = 0.0f; + if (expect_status(pernix_decompression_scale_f32(127.0f, bit_width, &decompression_scale_f32), + PERNIX_STATUS_OK) != 0 || + expect_status(pernix_compression_scale_f32(127.0f, bit_width, &compression_scale_f32), + PERNIX_STATUS_OK) != 0 || + expect_status(pernix_inverse_scale_f32(decompression_scale_f32, &inverse_scale_f32), + PERNIX_STATUS_OK) != 0) { + return 24; + } + if (fabsf(decompression_scale_f32 - 1.0f) > 0.0f || fabsf(compression_scale_f32 - 1.0f) > 0.0f || + fabsf(inverse_scale_f32 - 1.0f) > 0.0f) { + return 25; + } + if (expect_status(pernix_inverse_scale_f32(0.0f, &inverse_scale_f32), PERNIX_STATUS_INVALID_ARGUMENT) != 0 || + expect_status(pernix_compression_scale_f32(1.0f, 0, &compression_scale_f32), + PERNIX_STATUS_INVALID_ARGUMENT) != 0 || + expect_status(pernix_decompression_scale_f32(1.0f, bit_width, 0), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 26; + } + + double decompression_scale_f64 = 0.0; + double compression_scale_f64 = 0.0; + double inverse_scale_f64 = 0.0; + if (expect_status(pernix_decompression_scale_f64(127.0, bit_width, &decompression_scale_f64), + PERNIX_STATUS_OK) != 0 || + expect_status(pernix_compression_scale_f64(127.0, bit_width, &compression_scale_f64), + PERNIX_STATUS_OK) != 0 || + expect_status(pernix_inverse_scale_f64(decompression_scale_f64, &inverse_scale_f64), + PERNIX_STATUS_OK) != 0) { + return 27; + } + if (fabs(decompression_scale_f64 - 1.0) > 0.0 || fabs(compression_scale_f64 - 1.0) > 0.0 || + fabs(inverse_scale_f64 - 1.0) > 0.0) { + return 28; + } + + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, block_size, input_f32, 1.0f, compressed), + PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH) != 0) { + return 9; + } + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, 96, input_f32, 1.0f, compressed), + PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE) != 0) { + return 10; + } + if (expect_status(pernix_compress_block_f32((pernix_backend)255, bit_width, block_size, input_f32, 1.0f, + compressed), + PERNIX_STATUS_UNSUPPORTED_BACKEND) != 0) { + return 11; + } + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0f, compressed), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 12; + } + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 1.0f, 0), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 13; + } + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 0.0f, + compressed), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 14; + } + if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0f, + output_f32, true), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 15; + } + if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0f, 0, + true), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 16; + } + if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, -1.0f, + output_f32, true), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 17; + } + if (expect_status(pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 1.0f, + compressed, 0), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 18; + } + + if (expect_status(pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f64, 1.0, + compressed), + PERNIX_STATUS_OK) != 0) { + return 19; + } + if (expect_status(pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0, + output_f64, true), + PERNIX_STATUS_OK) != 0) { + return 20; + } + if (expect_status(pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0, compressed, + 1), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 21; + } + if (expect_status(pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0, 0, + 1, true), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 22; + } + + return 0; +} diff --git a/tests/deterministic_correctness_tests.cpp b/tests/deterministic_correctness_tests.cpp new file mode 100644 index 0000000..7265eca --- /dev/null +++ b/tests/deterministic_correctness_tests.cpp @@ -0,0 +1,385 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if PERNIX_TEST_BLOCK_SIZE == 64 +namespace { +constexpr u32 kBlockSize = 64; +constexpr u32 kBlocks = 3; + +enum class Pattern { + Zero, + Constant, + Increasing, + Decreasing, + SmallMagnitude, + LargeMagnitude, + QuantizationBoundary, + PseudoRandom +}; + +struct PatternCase { + Pattern pattern; + std::string_view name; +}; + +constexpr std::array kPatternCases{{ + {Pattern::Zero, "zero"}, + {Pattern::Constant, "constant"}, + {Pattern::Increasing, "increasing"}, + {Pattern::Decreasing, "decreasing"}, + {Pattern::SmallMagnitude, "small_magnitude"}, + {Pattern::LargeMagnitude, "large_magnitude"}, + {Pattern::QuantizationBoundary, "quantization_boundary"}, + {Pattern::PseudoRandom, "pseudo_random"}, +}}; + +constexpr std::array kComparisonBackends{{ + PERNIX_BACKEND_X86_AVX2, + PERNIX_BACKEND_X86_BMI2, + PERNIX_BACKEND_X86_AVX512_VBMI, + PERNIX_BACKEND_ARM64_NEON, + PERNIX_BACKEND_ARM64_SVE, +}}; + +template +FloatT abs_value(const FloatT value) { + return value < FloatT{0} ? -value : value; +} + +template +FloatT max_abs(const std::vector& values) { + FloatT bmax = FloatT{0}; + for (const FloatT value : values) { + bmax = std::max(bmax, abs_value(value)); + } + return bmax; +} + +template +pernix_status compute_scale(const FloatT bmax, const u8 bit_width, FloatT& scale) { + if constexpr (std::is_same_v) { + return pernix_scale_f32(bmax, bit_width, &scale); + } else { + return pernix_scale_f64(bmax, bit_width, &scale); + } +} + +template +FloatT tolerance_for_scale(const FloatT scale) { + return (abs_value(scale) * static_cast(0.55)) + + (std::numeric_limits::epsilon() * static_cast(64)); +} + +template +std::vector make_dataset(const Pattern pattern, const u8 bit_width, const u32 blocks = kBlocks) { + const u32 elements_per_block = pernix_elements_per_block(bit_width); + std::vector values(static_cast(elements_per_block) * blocks); + const bool one_bit = bit_width == 1; + const FloatT amplitude = [&] { + switch (pattern) { + case Pattern::SmallMagnitude: + return static_cast(1.0e-5); + case Pattern::LargeMagnitude: + return static_cast(1.0e6); + default: + return static_cast(16); + } + }(); + + std::mt19937 rng(0x5065726eU + bit_width); + std::uniform_real_distribution uniform(one_bit ? 0.0 : -1.0, 1.0); + + for (usize i = 0; i < values.size(); ++i) { + const FloatT unit = values.size() > 1 + ? static_cast((static_cast(i) / static_cast(values.size() - 1)) * + 2.0 - 1.0) + : FloatT{0}; + switch (pattern) { + case Pattern::Zero: + values[i] = FloatT{0}; + break; + case Pattern::Constant: + values[i] = one_bit ? amplitude : amplitude / FloatT{2}; + break; + case Pattern::Increasing: + values[i] = one_bit ? static_cast(i % 2U) * amplitude : unit * amplitude; + break; + case Pattern::Decreasing: + values[i] = one_bit ? static_cast((i + 1U) % 2U) * amplitude : -unit * amplitude; + break; + case Pattern::SmallMagnitude: + case Pattern::LargeMagnitude: + values[i] = one_bit ? static_cast(i % 2U) * amplitude + : static_cast(((static_cast(i % 9U) - 4) / 4.0)) * amplitude; + break; + case Pattern::QuantizationBoundary: { + const FloatT step = amplitude / static_cast(17); + if (one_bit) { + values[i] = static_cast(i % 2U) * step; + } else { + const int bucket = static_cast(i % 9U) - 4; + const FloatT nudge = (i % 2U == 0U) ? step * static_cast(0.49) + : step * static_cast(-0.49); + values[i] = (static_cast(bucket) * step) + nudge; + } + break; + } + case Pattern::PseudoRandom: + values[i] = static_cast(uniform(rng)) * amplitude; + break; + } + } + + return values; +} + +template +void expect_near_input(const std::vector& input, const std::vector& restored, const FloatT scale) { + ASSERT_EQ(restored.size(), input.size()); + const FloatT tolerance = tolerance_for_scale(scale); + for (usize i = 0; i < input.size(); ++i) { + ASSERT_NEAR(restored[i], input[i], tolerance) << "element=" << i << ", scale=" << scale; + } +} + +template +pernix_status compress_blocks(pernix_backend backend, const u8 bit_width, const std::vector& input, + const FloatT inverse_scale, std::vector& compressed, const u32 blocks) { + if constexpr (std::is_same_v) { + return pernix_compress_blocks_f32(backend, bit_width, kBlockSize, input.data(), inverse_scale, + compressed.data(), blocks); + } else { + return pernix_compress_blocks_f64(backend, bit_width, kBlockSize, input.data(), inverse_scale, + compressed.data(), blocks); + } +} + +template +pernix_status decompress_blocks(pernix_backend backend, const u8 bit_width, const std::vector& compressed, + const FloatT scale, std::vector& output, const u32 blocks, + const bool sign_values = true) { + if constexpr (std::is_same_v) { + return pernix_decompress_blocks_f32(backend, bit_width, kBlockSize, compressed.data(), scale, output.data(), + blocks, sign_values); + } else { + return pernix_decompress_blocks_f64(backend, bit_width, kBlockSize, compressed.data(), scale, output.data(), + blocks, sign_values); + } +} + +template +void expect_fallback_round_trip(const u8 bit_width, const PatternCase& pattern_case) { + SCOPED_TRACE(::testing::Message() << "bit_width=" << static_cast(bit_width) + << ", pattern=" << pattern_case.name); + + const auto input = make_dataset(pattern_case.pattern, bit_width); + FloatT scale = FloatT{0}; + ASSERT_EQ(compute_scale(max_abs(input), bit_width, scale), PERNIX_STATUS_OK); + + std::vector compressed(kBlocks * kBlockSize); + std::vector restored(input.size(), FloatT{}); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input, FloatT{1} / scale, compressed, kBlocks), + PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed, scale, restored, kBlocks), + PERNIX_STATUS_OK); + expect_near_input(input, restored, scale); +} + +template +void expect_backend_matches_fallback(const pernix_backend backend, const u8 bit_width) { + SCOPED_TRACE(::testing::Message() << "backend=" << static_cast(backend) + << ", bit_width=" << static_cast(bit_width)); + + const auto input = make_dataset(Pattern::PseudoRandom, bit_width, 2); + FloatT scale = FloatT{0}; + ASSERT_EQ(compute_scale(max_abs(input), bit_width, scale), PERNIX_STATUS_OK); + + std::vector fallback_compressed(2 * kBlockSize); + std::vector backend_compressed(2 * kBlockSize); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input, FloatT{1} / scale, fallback_compressed, 2), + PERNIX_STATUS_OK); + + const auto compress_status = compress_blocks(backend, bit_width, input, FloatT{1} / scale, backend_compressed, 2); + if (compress_status != PERNIX_STATUS_OK) { + EXPECT_TRUE(compress_status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || + compress_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || + compress_status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + return; + } + + std::vector fallback_restored(input.size(), FloatT{}); + std::vector backend_restored(input.size(), FloatT{}); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, fallback_compressed, scale, fallback_restored, 2), + PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, backend_compressed, scale, backend_restored, 2), + PERNIX_STATUS_OK); + + const FloatT tolerance = (abs_value(scale) * static_cast(1.1)) + + (std::numeric_limits::epsilon() * static_cast(64)); + for (usize i = 0; i < input.size(); ++i) { + ASSERT_NEAR(backend_restored[i], fallback_restored[i], tolerance) + << "element=" << i << ", scale=" << scale; + } + + std::vector backend_decompressed(input.size(), FloatT{}); + const auto decompress_status = + decompress_blocks(backend, bit_width, fallback_compressed, scale, backend_decompressed, 2); + if (decompress_status != PERNIX_STATUS_OK) { + EXPECT_TRUE(decompress_status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || + decompress_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || + decompress_status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + return; + } + for (usize i = 0; i < input.size(); ++i) { + ASSERT_NEAR(backend_decompressed[i], fallback_restored[i], tolerance) + << "element=" << i << ", scale=" << scale; + } +} +} // namespace + +TEST(DeterministicCorrectnessTest, F32FallbackRoundTripsAllPatternsAndBitWidths) { + for (u8 bit_width = pernix_min_bit_width(); bit_width <= pernix_max_bit_width(); ++bit_width) { + for (const PatternCase& pattern_case : kPatternCases) { + expect_fallback_round_trip(bit_width, pattern_case); + } + } +} + +TEST(DeterministicCorrectnessTest, F64FallbackRoundTripsAllPatternsAndBitWidths) { + for (u8 bit_width = pernix_min_bit_width(); bit_width <= pernix_max_bit_width(); ++bit_width) { + for (const PatternCase& pattern_case : kPatternCases) { + expect_fallback_round_trip(bit_width, pattern_case); + } + } +} + +TEST(DeterministicCorrectnessTest, SimdBackendsMatchFallbackWhereAvailable) { + for (const pernix_backend backend : kComparisonBackends) { + for (u8 bit_width = pernix_min_bit_width(); bit_width <= pernix_max_bit_width(); ++bit_width) { + expect_backend_matches_fallback(backend, bit_width); + expect_backend_matches_fallback(backend, bit_width); + } + } +} + +TEST(DeterministicCorrectnessTest, F32AndF64HaveConsistentFallbackSemantics) { + constexpr u8 bit_width = 12; + const u32 elements = pernix_elements_per_block(bit_width) * 2U; + std::vector input_f32(elements); + for (usize i = 0; i < input_f32.size(); ++i) { + input_f32[i] = static_cast(static_cast(i % 9U) - 4); + } + std::vector input_f64(input_f32.begin(), input_f32.end()); + + std::vector compressed_f32(2 * kBlockSize); + std::vector compressed_f64(2 * kBlockSize); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input_f32, 1.0f, compressed_f32, 2), + PERNIX_STATUS_OK); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input_f64, 1.0, compressed_f64, 2), + PERNIX_STATUS_OK); + EXPECT_EQ(compressed_f64, compressed_f32); + + std::vector restored_f32(input_f32.size()); + std::vector restored_f64(input_f64.size()); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed_f32, 1.0f, restored_f32, 2), + PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed_f64, 1.0, restored_f64, 2), + PERNIX_STATUS_OK); + + for (usize i = 0; i < restored_f32.size(); ++i) { + EXPECT_DOUBLE_EQ(static_cast(restored_f32[i]), restored_f64[i]); + } +} + +TEST(DeterministicCorrectnessTest, SignValuesFalseUsesUnsignedPackedValuesForAllBitWidths) { + for (u8 bit_width = pernix_min_bit_width(); bit_width <= pernix_max_bit_width(); ++bit_width) { + SCOPED_TRACE(::testing::Message() << "bit_width=" << static_cast(bit_width)); + std::vector packed(kBlockSize, 0xFFU); + const u32 elements = pernix_elements_per_block(bit_width); + std::vector signed_values(elements); + std::vector unsigned_values(elements); + + ASSERT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, + signed_values.data(), true), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, + unsigned_values.data(), false), + PERNIX_STATUS_OK); + + const float expected_unsigned = static_cast((1U << bit_width) - 1U); + EXPECT_EQ(unsigned_values[0], expected_unsigned); + if (bit_width == 1) { + EXPECT_EQ(signed_values[0], 1.0f); + } else { + EXPECT_EQ(signed_values[0], -1.0f); + } + } +} + +TEST(DeterministicCorrectnessTest, CppSpanWrappersMatchCApiOutputAndStatus) { + constexpr u8 bit_width = 10; + const auto input = make_dataset(Pattern::Increasing, bit_width, 1); + float scale = 0.0f; + ASSERT_EQ(compute_scale(max_abs(input), bit_width, scale), PERNIX_STATUS_OK); + + std::vector c_compressed(kBlockSize); + std::vector cpp_compressed(kBlockSize); + EXPECT_EQ(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, input.data(), 1.0f / scale, + c_compressed.data()), + PERNIX_STATUS_OK); + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), + 1.0f / scale, std::span(cpp_compressed)), + PERNIX_STATUS_OK); + EXPECT_EQ(cpp_compressed, c_compressed); + + std::vector c_restored(input.size()); + std::vector cpp_restored(input.size()); + EXPECT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, c_compressed.data(), scale, + c_restored.data(), true), + PERNIX_STATUS_OK); + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, bit_width, kBlockSize, + std::span(cpp_compressed), scale, std::span(cpp_restored)), + PERNIX_STATUS_OK); + EXPECT_EQ(cpp_restored, c_restored); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, 0, kBlockSize, std::span(input), + 1.0f / scale, std::span(cpp_compressed)), + pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, kBlockSize, input.data(), 1.0f / scale, + c_compressed.data())); +} + +TEST(DeterministicCorrectnessTest, SpanValidationRejectsUndersizedMultiBlockBuffers) { + constexpr u8 bit_width = 8; + const u32 elements = pernix_elements_per_block(bit_width) * 2U; + std::vector input(elements, 1.0f); + std::vector compressed(2U * kBlockSize); + std::vector restored(elements); + + EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, + std::span(input).first(input.size() - 1U), 1.0f, + std::span(compressed), 2), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), + 1.0f, std::span(compressed).first(compressed.size() - 1U), 2), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, + std::span(compressed).first(compressed.size() - 1U), 1.0f, + std::span(restored), 2), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, + std::span(compressed), 1.0f, + std::span(restored).first(restored.size() - 1U), 2), + PERNIX_STATUS_INVALID_ARGUMENT); +} +#endif diff --git a/tests/install_consumer/CMakeLists.txt b/tests/install_consumer/CMakeLists.txt new file mode 100644 index 0000000..83ed283 --- /dev/null +++ b/tests/install_consumer/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.24) +project(pernix_install_consumer LANGUAGES C CXX) + +find_package(pernix CONFIG REQUIRED) + +add_executable(pernix_consumer_cpp main.cpp) +target_link_libraries(pernix_consumer_cpp PRIVATE pernix::pernix) + +add_executable(pernix_consumer_c main.c) +target_link_libraries(pernix_consumer_c PRIVATE pernix::pernix) + +enable_testing() +add_test(NAME pernix_consumer_cpp COMMAND pernix_consumer_cpp) +add_test(NAME pernix_consumer_c COMMAND pernix_consumer_c) diff --git a/tests/install_consumer/main.c b/tests/install_consumer/main.c new file mode 100644 index 0000000..62c324b --- /dev/null +++ b/tests/install_consumer/main.c @@ -0,0 +1,28 @@ +#include + +int main(void) { + enum { bit_width = 8, block_size = 64, elements = (block_size * 8) / bit_width }; + float input[elements]; + float restored[elements]; + u8 compressed[block_size]; + + for (int i = 0; i < elements; ++i) { + input[i] = (float)((i % 17) - 8) * 0.125f; + } + + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 8.0f, + compressed) != PERNIX_STATUS_OK) { + return 1; + } + if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 0.125f, + restored, true) != PERNIX_STATUS_OK) { + return 2; + } + + for (int i = 0; i < elements; ++i) { + if (input[i] != restored[i]) { + return 3; + } + } + return 0; +} diff --git a/tests/install_consumer/main.cpp b/tests/install_consumer/main.cpp new file mode 100644 index 0000000..7300997 --- /dev/null +++ b/tests/install_consumer/main.cpp @@ -0,0 +1,35 @@ +#include + +#include +#include +#include + +int main() { + constexpr u8 bit_width = 8; + constexpr u32 block_size = 64; + constexpr std::size_t elements = (block_size * 8U) / bit_width; + + std::array input{}; + for (std::size_t i = 0; i < input.size(); ++i) { + input[i] = static_cast(static_cast(i % 17U) - 8) * 0.125f; + } + + std::array compressed{}; + std::array restored{}; + + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, 8.0f, compressed) != + PERNIX_STATUS_OK) { + return 1; + } + if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, 0.125f, restored) != + PERNIX_STATUS_OK) { + return 2; + } + + for (std::size_t i = 0; i < input.size(); ++i) { + if (std::abs(input[i] - restored[i]) > 0.0f) { + return 3; + } + } + return 0; +} diff --git a/tests/install_consumer/run.cmake b/tests/install_consumer/run.cmake new file mode 100644 index 0000000..cfa286f --- /dev/null +++ b/tests/install_consumer/run.cmake @@ -0,0 +1,64 @@ +if (NOT DEFINED PERNIX_BUILD_DIR) + message(FATAL_ERROR "PERNIX_BUILD_DIR is required") +endif () +if (NOT DEFINED PERNIX_CONSUMER_SOURCE_DIR) + message(FATAL_ERROR "PERNIX_CONSUMER_SOURCE_DIR is required") +endif () +if (NOT DEFINED PERNIX_CONSUMER_BUILD_DIR) + message(FATAL_ERROR "PERNIX_CONSUMER_BUILD_DIR is required") +endif () +if (NOT DEFINED PERNIX_INSTALL_PREFIX) + message(FATAL_ERROR "PERNIX_INSTALL_PREFIX is required") +endif () + +set(PERNIX_CONFIG_ARG) +if (DEFINED PERNIX_TEST_CONFIG AND NOT PERNIX_TEST_CONFIG STREQUAL "") + list(APPEND PERNIX_CONFIG_ARG --config "${PERNIX_TEST_CONFIG}") +endif () + +file(REMOVE_RECURSE "${PERNIX_INSTALL_PREFIX}" "${PERNIX_CONSUMER_BUILD_DIR}") + +execute_process( + COMMAND "${CMAKE_COMMAND}" --install "${PERNIX_BUILD_DIR}" --prefix "${PERNIX_INSTALL_PREFIX}" ${PERNIX_CONFIG_ARG} + RESULT_VARIABLE PERNIX_INSTALL_RESULT +) +if (NOT PERNIX_INSTALL_RESULT EQUAL 0) + message(FATAL_ERROR "Installing pernix for consumer test failed") +endif () + +set(PERNIX_COMPILER_ARGS) +if (DEFINED PERNIX_C_COMPILER AND NOT PERNIX_C_COMPILER STREQUAL "") + list(APPEND PERNIX_COMPILER_ARGS "-DCMAKE_C_COMPILER=${PERNIX_C_COMPILER}") +endif () +if (DEFINED PERNIX_CXX_COMPILER AND NOT PERNIX_CXX_COMPILER STREQUAL "") + list(APPEND PERNIX_COMPILER_ARGS "-DCMAKE_CXX_COMPILER=${PERNIX_CXX_COMPILER}") +endif () + +execute_process( + COMMAND "${CMAKE_COMMAND}" + -S "${PERNIX_CONSUMER_SOURCE_DIR}" + -B "${PERNIX_CONSUMER_BUILD_DIR}" + "-DCMAKE_PREFIX_PATH=${PERNIX_INSTALL_PREFIX}" + "-DCMAKE_BUILD_TYPE=Release" + ${PERNIX_COMPILER_ARGS} + RESULT_VARIABLE PERNIX_CONFIGURE_RESULT +) +if (NOT PERNIX_CONFIGURE_RESULT EQUAL 0) + message(FATAL_ERROR "Configuring installed pernix consumer failed") +endif () + +execute_process( + COMMAND "${CMAKE_COMMAND}" --build "${PERNIX_CONSUMER_BUILD_DIR}" --config Release --parallel 12 + RESULT_VARIABLE PERNIX_BUILD_RESULT +) +if (NOT PERNIX_BUILD_RESULT EQUAL 0) + message(FATAL_ERROR "Building installed pernix consumer failed") +endif () + +execute_process( + COMMAND "${CMAKE_CTEST_COMMAND}" --test-dir "${PERNIX_CONSUMER_BUILD_DIR}" --output-on-failure + RESULT_VARIABLE PERNIX_TEST_RESULT +) +if (NOT PERNIX_TEST_RESULT EQUAL 0) + message(FATAL_ERROR "Running installed pernix consumer tests failed") +endif () From 2045935c3b39e923bf2865122da9a8e0f2276527 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 9 Jul 2026 11:57:26 +0200 Subject: [PATCH 27/43] feat: add AddressSanitizer and UndefinedBehaviorSanitizer options to CMake configuration --- .github/workflows/ci.yml | 234 +++++++++++++++++++++++++++++++++++++++ CMakeLists.txt | 23 ++++ 2 files changed, 257 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e15764f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,234 @@ +name: CI + +on: + push: + pull_request: + +env: + CCACHE_DIR: /home/runner/.cache/ccache + CCACHE_MAXSIZE: 500M + +jobs: + release: + name: ${{ matrix.compiler }} release + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + compiler: [gcc, clang] + include: + - compiler: gcc + cc: gcc + cxx: g++ + - compiler: clang + cc: clang + cxx: clang++ + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ clang + + - name: Cache ccache + uses: actions/cache@v4 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-${{ matrix.compiler }}-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-${{ matrix.compiler }}-${{ github.job }}- + ccache-${{ runner.os }}-${{ matrix.compiler }}- + ccache-${{ runner.os }}- + + - name: Configure + run: | + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=${{ matrix.cc }} \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_INSTALL=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF + + - name: Build + run: cmake --build build --config Release -j"$(nproc)" + + - name: Test + run: ctest --test-dir build --output-on-failure + + - name: Run examples + run: | + ./build/examples/pernix_example_cpp + ./build/examples/pernix_example_c + + fallback-only: + name: fallback-only release + runs-on: ubuntu-latest + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ + + - name: Cache ccache + uses: actions/cache@v4 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-gcc-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-gcc-${{ github.job }}- + ccache-${{ runner.os }}-gcc- + ccache-${{ runner.os }}- + + - name: Configure + run: | + cmake -S . -B build-fallback -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_INSTALL=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_ENABLE_X86_AVX2=OFF \ + -DPERNIX_ENABLE_X86_BMI2=OFF \ + -DPERNIX_ENABLE_X86_AVX512VBMI=OFF \ + -DPERNIX_ENABLE_ARM64_NEON=OFF \ + -DPERNIX_ENABLE_ARM64_SVE2=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build + run: cmake --build build-fallback --config Release -j"$(nproc)" + + - name: Test + run: ctest --test-dir build-fallback --output-on-failure + + - name: Run examples + run: | + ./build-fallback/examples/pernix_example_cpp + ./build-fallback/examples/pernix_example_c + + debug-and-sanitizers: + name: debug and sanitizers + runs-on: ubuntu-latest + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ + + - name: Cache ccache + uses: actions/cache@v4 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-gcc-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-gcc-${{ github.job }}- + ccache-${{ runner.os }}-gcc- + ccache-${{ runner.os }}- + + - name: Configure debug + run: | + cmake -S . -B build-debug -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_INSTALL=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build debug + run: cmake --build build-debug --config Debug -j"$(nproc)" + + - name: Test debug + run: ctest --test-dir build-debug --output-on-failure + + - name: Configure sanitizers + run: | + cmake -S . -B build-sanitizers -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=OFF \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=OFF \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_ENABLE_SANITIZER_ADDRESS=ON \ + -DPERNIX_ENABLE_SANITIZER_UNDEFINED=ON \ + -DPERNIX_ENABLE_X86_AVX2=OFF \ + -DPERNIX_ENABLE_X86_BMI2=OFF \ + -DPERNIX_ENABLE_X86_AVX512VBMI=OFF \ + -DPERNIX_ENABLE_ARM64_NEON=OFF \ + -DPERNIX_ENABLE_ARM64_SVE2=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build sanitizers + run: cmake --build build-sanitizers --config Debug -j"$(nproc)" + + - name: Test sanitizers + run: ctest --test-dir build-sanitizers --output-on-failure + + fortran: + name: Fortran bindings + runs-on: ubuntu-latest + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ gfortran + + - name: Cache ccache + uses: actions/cache@v4 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-gfortran-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-gfortran-${{ github.job }}- + ccache-${{ runner.os }}-gfortran- + ccache-${{ runner.os }}- + + - name: Configure + run: | + cmake -S . -B build-fortran -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_Fortran_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=OFF \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build + run: cmake --build build-fortran --config Release -j"$(nproc)" + + - name: Test + run: ctest --test-dir build-fortran --output-on-failure + + - name: Run Fortran example + run: ./build-fortran/examples/pernix_example_fortran diff --git a/CMakeLists.txt b/CMakeLists.txt index 982278f..f77fc88 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,8 @@ option(BUILD_SHARED_LIBS "Build pernix as a shared library by default" ON) option(PERNIX_ENABLE_TESTS "Build pernix unit and integration tests" ON) option(PERNIX_ENABLE_CODE_COVERAGE "Enable code coverage for pernix tests (requires lcov/genhtml)" OFF) option(PERNIX_ENABLE_LTO "Enable interprocedural optimization/link-time optimization" OFF) +option(PERNIX_ENABLE_SANITIZER_ADDRESS "Build pernix targets with AddressSanitizer instrumentation" OFF) +option(PERNIX_ENABLE_SANITIZER_UNDEFINED "Build pernix targets with UndefinedBehaviorSanitizer instrumentation" OFF) option(PERNIX_ENABLE_INSTALL "Install pernix library, headers, and package metadata" ON) option(PERNIX_ENABLE_DOXYGEN "Build API documentation with Doxygen" OFF) option(PERNIX_INSTALL_DOCS "Install generated Doxygen documentation; requires PERNIX_ENABLE_DOXYGEN=ON" OFF) @@ -100,6 +102,7 @@ set(PERNIX_PRIVATE_COMPILE_OPTIONS) foreach (PERNIX_CXX_FLAG -Wall -Wextra + -Wpedantic -Wshadow -Wfloat-equal -Wold-style-cast @@ -116,6 +119,26 @@ foreach (PERNIX_CXX_FLAG endif () endforeach () +set(PERNIX_SANITIZER_OPTIONS) +if (PERNIX_ENABLE_SANITIZER_ADDRESS OR PERNIX_ENABLE_SANITIZER_UNDEFINED) + if (NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|IntelLLVM") + message(FATAL_ERROR "PERNIX sanitizer options are supported only with GCC/Clang-like compilers") + endif () +endif () + +if (PERNIX_ENABLE_SANITIZER_ADDRESS) + list(APPEND PERNIX_SANITIZER_OPTIONS "-fsanitize=address") +endif () + +if (PERNIX_ENABLE_SANITIZER_UNDEFINED) + list(APPEND PERNIX_SANITIZER_OPTIONS "-fsanitize=undefined") +endif () + +if (PERNIX_SANITIZER_OPTIONS) + add_compile_options(${PERNIX_SANITIZER_OPTIONS}) + add_link_options(${PERNIX_SANITIZER_OPTIONS}) +endif () + if (PERNIX_ENABLE_LTO) include(CheckIPOSupported) check_ipo_supported(RESULT PERNIX_IPO_SUPPORTED OUTPUT PERNIX_IPO_ERROR) From 917d430b97b2242d59473a4d9ca5d6202f8d720a Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 9 Jul 2026 12:01:46 +0200 Subject: [PATCH 28/43] feat: update CI configuration to include main and release branches for push and pull_request events --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e15764f..52f83f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,15 @@ name: CI on: push: + branches: + - main + - "release/**" + tags: + - "v*" pull_request: + branches: + - main + - "release/**" env: CCACHE_DIR: /home/runner/.cache/ccache From ad146826c877d1f0e7fe68bbd163cbbfeaf458b2 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 9 Jul 2026 12:58:02 +0200 Subject: [PATCH 29/43] feat: update CI configuration for arm64 support and upgrade cache action to v6 --- .github/workflows/ci.yml | 62 +++++++++++++++++++++++++++++--- CMakeLists.txt | 32 ----------------- README.md | 4 +-- bindings/README.md | 3 +- src/CMakeLists.txt | 4 +-- tests/install_consumer/run.cmake | 7 +++- 6 files changed, 70 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52f83f4..4dfd4bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ clang - name: Cache ccache - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: /home/runner/.cache/ccache key: ccache-${{ runner.os }}-${{ matrix.compiler }}-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} @@ -76,6 +76,60 @@ jobs: ./build/examples/pernix_example_cpp ./build/examples/pernix_example_c + arm64-release: + name: arm64 release + runs-on: ubuntu-24.04-arm + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ + + - name: Show runner CPU + run: | + uname -m + lscpu + grep -m1 Features /proc/cpuinfo || true + + - name: Cache ccache + uses: actions/cache@v6 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-arm64-gcc-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-arm64-gcc-${{ github.job }}- + ccache-${{ runner.os }}-arm64-gcc- + ccache-${{ runner.os }}-arm64- + ccache-${{ runner.os }}- + + - name: Configure + run: | + cmake -S . -B build-arm64 -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_INSTALL=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build + run: cmake --build build-arm64 --config Release -j"$(nproc)" + + - name: Test + run: ctest --test-dir build-arm64 --output-on-failure + + - name: Run examples + run: | + ./build-arm64/examples/pernix_example_cpp + ./build-arm64/examples/pernix_example_c + fallback-only: name: fallback-only release runs-on: ubuntu-latest @@ -90,7 +144,7 @@ jobs: sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ - name: Cache ccache - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: /home/runner/.cache/ccache key: ccache-${{ runner.os }}-gcc-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} @@ -142,7 +196,7 @@ jobs: sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ - name: Cache ccache - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: /home/runner/.cache/ccache key: ccache-${{ runner.os }}-gcc-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} @@ -209,7 +263,7 @@ jobs: sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ gfortran - name: Cache ccache - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: /home/runner/.cache/ccache key: ccache-${{ runner.os }}-gfortran-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} diff --git a/CMakeLists.txt b/CMakeLists.txt index f77fc88..0c88462 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,38 +43,6 @@ if (PERNIX_ENABLE_INSTALL_CONSUMER_TESTS AND PERNIX_ENABLE_TESTS AND NOT PERNIX_ message(FATAL_ERROR "PERNIX_ENABLE_INSTALL_CONSUMER_TESTS requires PERNIX_ENABLE_INSTALL=ON") endif () -#[===[ -set(PERNIX_BUNDLE_SIMDE_FOR_INSTALL OFF) -if (PERNIX_USE_SIMDE) - if (PERNIX_SIMDE_PROVIDER STREQUAL "AUTO" OR PERNIX_SIMDE_PROVIDER STREQUAL "PACKAGE") - find_package(simde CONFIG QUIET) - endif () - - if (NOT TARGET simde::simde AND (PERNIX_SIMDE_PROVIDER STREQUAL "AUTO" OR PERNIX_SIMDE_PROVIDER STREQUAL "FETCH")) - include(FetchContent) - set(SIMDE_TEST_CMAKE_PACKAGING OFF CACHE BOOL "Test SIMDe CMake packaging" FORCE) - FetchContent_Declare( - simde - GIT_REPOSITORY https://github.com/simd-everywhere/simde.git - GIT_TAG f3e8262173b7089db9a9d57a9ecef8dd07ad9c97 - GIT_PROGRESS TRUE - EXCLUDE_FROM_ALL - ) - FetchContent_MakeAvailable(simde) - set(PERNIX_BUNDLE_SIMDE_FOR_INSTALL ON) - endif () - - if (NOT TARGET simde::simde AND DEFINED simde_SOURCE_DIR AND EXISTS "${simde_SOURCE_DIR}/simde") - add_library(simde::simde INTERFACE IMPORTED GLOBAL) - target_include_directories(simde::simde INTERFACE "${simde_SOURCE_DIR}") - endif () - - if (NOT TARGET simde::simde) - message(FATAL_ERROR "PERNIX_USE_SIMDE is enabled, but simde::simde was not found. Set PERNIX_SIMDE_PROVIDER=FETCH or install SIMDe's CMake package.") - endif () -endif () -]===] - include(CTest) include(GitVersion) get_git_version(GIT_VERSION) diff --git a/README.md b/README.md index 434a27a..9930017 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,7 @@ Release build: ```bash cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -cmake --build build --config Release -j12 +cmake --build build --config Release -j"$(nproc)" ``` Common options: @@ -260,7 +260,7 @@ target_link_libraries(my_target PRIVATE pernix::pernix) ```bash cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DPERNIX_ENABLE_TESTS=ON -DPERNIX_ENABLE_EXAMPLES=ON -cmake --build build --config Release -j12 +cmake --build build --config Release -j"$(nproc)" ctest --test-dir build --output-on-failure ``` diff --git a/bindings/README.md b/bindings/README.md index d5c0273..7866dad 100644 --- a/bindings/README.md +++ b/bindings/README.md @@ -8,7 +8,8 @@ such as `pernix_compress_block_f32`, `pernix_decompress_block_f32`, and the matc To build the Fortran bindings, you need a Fortran compiler. Enable them from the main CMake build: ```bash -cmake -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON .. +cmake -S . -B build -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON +cmake --build build --config Release -j"$(nproc)" ``` The `pernix_bindings_fortran_test` target performs one f32 64-byte block round trip through the Fortran modules. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 74ab4b8..be7dd5d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -22,8 +22,8 @@ else () endif () target_sources(pernix PRIVATE ${PERNIX_GENERIC_SOURCE_FILES}) -target_compile_features(pernix PUBLIC cxx_std_23) -target_compile_features(pernix_header_only INTERFACE cxx_std_23) +target_compile_features(pernix PUBLIC cxx_std_20) +target_compile_features(pernix_header_only INTERFACE cxx_std_20) target_compile_options(pernix PRIVATE ${PERNIX_PRIVATE_COMPILE_OPTIONS}) set_target_properties(pernix PROPERTIES diff --git a/tests/install_consumer/run.cmake b/tests/install_consumer/run.cmake index cfa286f..763a8e5 100644 --- a/tests/install_consumer/run.cmake +++ b/tests/install_consumer/run.cmake @@ -34,6 +34,11 @@ if (DEFINED PERNIX_CXX_COMPILER AND NOT PERNIX_CXX_COMPILER STREQUAL "") list(APPEND PERNIX_COMPILER_ARGS "-DCMAKE_CXX_COMPILER=${PERNIX_CXX_COMPILER}") endif () +cmake_host_system_information(RESULT PERNIX_PARALLEL_JOBS QUERY NUMBER_OF_LOGICAL_CORES) +if (NOT PERNIX_PARALLEL_JOBS OR PERNIX_PARALLEL_JOBS LESS 1) + set(PERNIX_PARALLEL_JOBS 1) +endif () + execute_process( COMMAND "${CMAKE_COMMAND}" -S "${PERNIX_CONSUMER_SOURCE_DIR}" @@ -48,7 +53,7 @@ if (NOT PERNIX_CONFIGURE_RESULT EQUAL 0) endif () execute_process( - COMMAND "${CMAKE_COMMAND}" --build "${PERNIX_CONSUMER_BUILD_DIR}" --config Release --parallel 12 + COMMAND "${CMAKE_COMMAND}" --build "${PERNIX_CONSUMER_BUILD_DIR}" --config Release --parallel "${PERNIX_PARALLEL_JOBS}" RESULT_VARIABLE PERNIX_BUILD_RESULT ) if (NOT PERNIX_BUILD_RESULT EQUAL 0) From b90848ebf3833a4ad0874fd1ecaed875c00e387f Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 9 Jul 2026 14:30:26 +0200 Subject: [PATCH 30/43] feat: update CI configuration and improve error handling in fallback tests --- .github/workflows/ci.yml | 5 +++++ tests/fallback_tests.cpp | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dfd4bf..f1547cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -216,6 +216,11 @@ jobs: -DPERNIX_ENABLE_INSTALL=ON \ -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON \ -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_ENABLE_X86_AVX2=OFF \ + -DPERNIX_ENABLE_X86_BMI2=OFF \ + -DPERNIX_ENABLE_X86_AVX512VBMI=OFF \ + -DPERNIX_ENABLE_ARM64_NEON=OFF \ + -DPERNIX_ENABLE_ARM64_SVE2=OFF \ -DPERNIX_TEST_BLOCK_SIZES=64 - name: Build debug diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp index 44eedd6..86840a2 100644 --- a/tests/fallback_tests.cpp +++ b/tests/fallback_tests.cpp @@ -835,7 +835,7 @@ TEST(ErrorCodeTest, FallbackSimdReturnsUnsupportedImplementationByDefault) { EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); } -TEST(ErrorCodeTest, UnsupportedImplementationReturnsError) { +TEST(ErrorCodeTest, ExplicitUnavailableBackendReturnsError) { constexpr u32 BS = 64; constexpr u8 BW = 8; @@ -877,5 +877,7 @@ TEST(ErrorCodeTest, UnsupportedImplementationReturnsError) { } const auto st = pernix_compress_block_f32(backend, BW, BS, src, 1.0f, dst); - EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); + // A backend can be absent from the build, or compiled but unsupported by this CPU. + EXPECT_TRUE(st == PERNIX_STATUS_UNSUPPORTED_BACKEND || + st == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); } From 63ffb3e8294f0007d2512d83de54787e90cb78f5 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 9 Jul 2026 22:37:56 +0200 Subject: [PATCH 31/43] feat: refactor fallback compression and decompression functions for improved type safety and performance --- include/pernix/dispatch/select_impl.h | 4758 +++++++++-------- include/pernix/fallback/common.h | 51 +- include/pernix/fallback/scalar_compression.h | 212 +- .../pernix/fallback/scalar_decompression.h | 191 +- src/fallback/fallback_compression.cpp | 260 +- src/fallback/fallback_decompression.cpp | 286 +- 6 files changed, 2794 insertions(+), 2964 deletions(-) diff --git a/include/pernix/dispatch/select_impl.h b/include/pernix/dispatch/select_impl.h index 8cc578a..0ca59f1 100644 --- a/include/pernix/dispatch/select_impl.h +++ b/include/pernix/dispatch/select_impl.h @@ -33,943 +33,901 @@ #include #endif - namespace pernix::internal { - inline bool is_optional_fallback_backend(const Backend backend) { - switch (backend) { - case Backend::FallbackStdpar: - case Backend::FallbackSimd: - return true; - default: - return false; - } +inline bool is_optional_fallback_backend(const Backend backend) { + switch (backend) { + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + default: + return false; } +} - inline bool is_compiled_backend(const Backend backend) { - switch (backend) { - case Backend::Auto: - case Backend::Fallback: - return true; +inline bool is_compiled_backend(const Backend backend) { + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + return true; #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return true; + case Backend::FallbackStdpar: + return true; #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return true; + case Backend::FallbackSimd: + return true; #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return true; + case Backend::X86Avx2: + return true; #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return true; + case Backend::X86Bmi2: + return true; #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return true; + case Backend::X86Avx512Vbmi: + return true; #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return true; + case Backend::Arm64Neon: + return true; #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return true; -#endif - default: - return false; - } - } - - inline bool is_backend_supported_on_machine(const Backend backend) { - const CpuFeatures features = get_cached_cpu_features(); - - switch (backend) { - case Backend::Auto: - case Backend::Fallback: - case Backend::FallbackStdpar: - case Backend::FallbackSimd: - return true; - case Backend::X86Avx2: - return features.avx2; - case Backend::X86Bmi2: - return features.avx2 && features.bmi2; - case Backend::X86Avx512Vbmi: - return features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && - features.avx512vbmi; - case Backend::Arm64Neon: - return features.neon; - case Backend::Arm64Sve: - return features.sve2; - default: - return false; - } - } - - inline Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_compress_block_f32(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_block_f32(bit_width, block_size); + case Backend::Arm64Sve: + return true; +#endif + default: + return false; + } +} + +inline bool is_backend_supported_on_machine(const Backend backend) { + const CpuFeatures features = get_cached_cpu_features(); + + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + case Backend::X86Avx2: + return features.avx2; + case Backend::X86Bmi2: + return features.avx2 && features.bmi2; + case Backend::X86Avx512Vbmi: + return features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi; + case Backend::Arm64Neon: + return features.neon; + case Backend::Arm64Sve: + return features.sve2; + default: + return false; + } +} + +inline Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f32(bit_width, block_size); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_compress_block_f32(bit_width, block_size); + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_compress_block_f32(bit_width, block_size); + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_block_f32(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_block_f32(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_block_f32(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_block_f32(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_compress_block_f32(bit_width, block_size); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f32(bit_width, block_size); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - inline Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_compress_blocks_f32(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_blocks_f32(bit_width, block_size); +inline Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f32(bit_width, block_size); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_compress_blocks_f32(bit_width, block_size); + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_compress_blocks_f32(bit_width, block_size); + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_blocks_f32(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_blocks_f32(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_blocks_f32(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_compress_blocks_f32(bit_width, block_size); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f32(bit_width, block_size); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - inline Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_compress_block_f64(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_block_f64(bit_width, block_size); +inline Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f64(bit_width, block_size); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_compress_block_f64(bit_width, block_size); + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_compress_block_f64(bit_width, block_size); + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_block_f64(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_block_f64(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_block_f64(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_block_f64(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_compress_block_f64(bit_width, block_size); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f64(bit_width, block_size); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - inline Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_compress_blocks_f64(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_blocks_f64(bit_width, block_size); +inline Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f64(bit_width, block_size); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_compress_blocks_f64(bit_width, block_size); + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_compress_blocks_f64(bit_width, block_size); + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_blocks_f64(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_blocks_f64(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_blocks_f64(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_compress_blocks_f64(bit_width, block_size); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f64(bit_width, block_size); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - inline Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, - bool sign_values) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_decompress_block_f32(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +inline Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - inline Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, - bool sign_values) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +inline Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - inline Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, - bool sign_values) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_decompress_block_f64(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +inline Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - inline Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, - bool sign_values) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +inline Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - inline Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +inline Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif - return select_fallback_compress_block_f32(bit_width, block_size); - } + return select_fallback_compress_block_f32(bit_width, block_size); +} - inline Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +inline Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif - return select_fallback_compress_blocks_f32(bit_width, block_size); - } + return select_fallback_compress_blocks_f32(bit_width, block_size); +} - inline Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +inline Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif - return select_fallback_compress_block_f64(bit_width, block_size); - } + return select_fallback_compress_block_f64(bit_width, block_size); +} - inline Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +inline Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif - return select_fallback_compress_blocks_f64(bit_width, block_size); - } + return select_fallback_compress_blocks_f64(bit_width, block_size); +} - inline Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +inline Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif - return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); - } + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +} - inline Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +inline Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif - return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); - } + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +} - inline Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +inline Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif - return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); - } + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +} - inline Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +inline Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif - return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); - } + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); } +} // namespace pernix::internal namespace pernix::internal { #define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ -case N: return Kernel("fallback", &compress_block_fallback) + case N: \ + return Kernel("fallback", &compress_block_fallback) #define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ -case N: return Kernel("fallback", &compress_blocks_fallback) + case N: \ + return Kernel("fallback", &compress_blocks_fallback) #define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ -case N: return Kernel("fallback", &compress_block_fallback) + case N: \ + return Kernel("fallback", &compress_block_fallback) #define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ -case N: return Kernel("fallback", &compress_blocks_fallback) - -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + case N: \ + return Kernel("fallback", &compress_blocks_fallback) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ @@ -985,51 +943,53 @@ case N: return Kernel("fallback", &compress_blocks_fallback PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ - default: return {"fallback", nullptr}; \ + default: \ + return {"fallback", nullptr}; \ } #define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ - default: return {"fallback", nullptr}; \ - } - -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ @@ -1045,131 +1005,133 @@ case N: return Kernel("fallback", &compress_blocks_fallback PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ - default: return {"fallback", nullptr}; \ + default: \ + return {"fallback", nullptr}; \ } #define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ - default: return {"fallback", nullptr}; \ - } - - inline Kernel select_fallback_compress_block_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"fallback", nullptr}; - } - } - - inline Kernel select_fallback_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"fallback", nullptr}; - } - } - - inline Kernel select_fallback_compress_block_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"fallback", nullptr}; - } - } - - inline Kernel select_fallback_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"fallback", nullptr}; - } - } - - inline Kernel select_fallback_scalar_compress_block_f32(const u8 bit_width, - const u32 block_size) { - return select_fallback_compress_block_f32(bit_width, block_size); - } - - inline Kernel select_fallback_scalar_compress_blocks_f32(const u8 bit_width, - const u32 block_size) { - return select_fallback_compress_blocks_f32(bit_width, block_size); - } - - inline Kernel select_fallback_scalar_compress_block_f64(const u8 bit_width, - const u32 block_size) { - return select_fallback_compress_block_f64(bit_width, block_size); - } - - inline Kernel select_fallback_scalar_compress_blocks_f64(const u8 bit_width, - const u32 block_size) { - return select_fallback_compress_blocks_f64(bit_width, block_size); - } - - #define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(N, BS) \ - case N: return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) - - #define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(N, BS) \ - case N: return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) - - #define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(N, BS) \ - case N: return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) - - #define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(N, BS) \ - case N: return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) - - #define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(9, BS); \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +inline Kernel select_fallback_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_scalar_compress_block_f32(const u8 bit_width, const u32 block_size) { + return select_fallback_compress_block_f32(bit_width, block_size); +} + +inline Kernel select_fallback_scalar_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + return select_fallback_compress_blocks_f32(bit_width, block_size); +} + +inline Kernel select_fallback_scalar_compress_block_f64(const u8 bit_width, const u32 block_size) { + return select_fallback_compress_block_f64(bit_width, block_size); +} + +inline Kernel select_fallback_scalar_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + return select_fallback_compress_blocks_f64(bit_width, block_size); +} + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(12, BS); \ @@ -1185,51 +1147,53 @@ case N: return Kernel("fallback", &compress_blocks_fallback PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(24, BS); \ - default: return {"fallback_stdpar", nullptr}; \ - } - - #define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(24, BS); \ - default: return {"fallback_stdpar", nullptr}; \ - } - - #define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(9, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(12, BS); \ @@ -1245,130 +1209,136 @@ case N: return Kernel("fallback", &compress_blocks_fallback PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(24, BS); \ - default: return {"fallback_stdpar", nullptr}; \ - } - - #define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(24, BS); \ - default: return {"fallback_stdpar", nullptr}; \ - } - - inline Kernel select_fallback_stdpar_compress_block_f32(const u8 bit_width, const u32 block_size) { + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +inline Kernel select_fallback_stdpar_compress_block_f32(const u8 bit_width, const u32 block_size) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - switch (block_size) { - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(1024); - default: return {"fallback_stdpar", nullptr}; - } + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_stdpar_compress_blocks_f32(const u8 bit_width, const u32 block_size) { +inline Kernel select_fallback_stdpar_compress_blocks_f32(const u8 bit_width, const u32 block_size) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - switch (block_size) { - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(1024); - default: return {"fallback_stdpar", nullptr}; - } + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_stdpar_compress_block_f64(const u8 bit_width, const u32 block_size) { +inline Kernel select_fallback_stdpar_compress_block_f64(const u8 bit_width, const u32 block_size) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - switch (block_size) { - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(1024); - default: return {"fallback_stdpar", nullptr}; - } + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_stdpar_compress_blocks_f64(const u8 bit_width, const u32 block_size) { +inline Kernel select_fallback_stdpar_compress_blocks_f64(const u8 bit_width, const u32 block_size) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - switch (block_size) { - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(1024); - default: return {"fallback_stdpar", nullptr}; - } + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_simd_compress_block_f32(const u8, const u32) { +inline Kernel select_fallback_simd_compress_block_f32(const u8, const u32) { #if defined(PERNIX_BUILD_FALLBACK_SIMD) - return {"fallback_simd", &compress_block_fallback_simd<1, 64, f32>}; + return {"fallback_simd", &compress_block_fallback_simd<1, 64, f32>}; #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_simd_compress_blocks_f32(const u8, const u32) { +inline Kernel select_fallback_simd_compress_blocks_f32(const u8, const u32) { #if defined(PERNIX_BUILD_FALLBACK_SIMD) - return {"fallback_simd", &compress_blocks_fallback_simd<1, 64, f32>}; + return {"fallback_simd", &compress_blocks_fallback_simd<1, 64, f32>}; #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_simd_compress_block_f64(const u8, const u32) { +inline Kernel select_fallback_simd_compress_block_f64(const u8, const u32) { #if defined(PERNIX_BUILD_FALLBACK_SIMD) - return {"fallback_simd", &compress_block_fallback_simd<1, 64, f64>}; + return {"fallback_simd", &compress_block_fallback_simd<1, 64, f64>}; #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_simd_compress_blocks_f64(const u8, const u32) { +inline Kernel select_fallback_simd_compress_blocks_f64(const u8, const u32) { #if defined(PERNIX_BUILD_FALLBACK_SIMD) - return {"fallback_simd", &compress_blocks_fallback_simd<1, 64, f64>}; + return {"fallback_simd", &compress_blocks_fallback_simd<1, 64, f64>}; #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 @@ -1378,41 +1348,40 @@ case N: return Kernel("fallback", &compress_blocks_fallback #undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 #undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 -} +} // namespace pernix::internal namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ - return Kernel("fallback", &decompress_block_fallback) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ - return Kernel("fallback", &decompress_blocks_fallback) - - -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ - return Kernel("fallback", &decompress_block_fallback) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ - return Kernel("fallback", &decompress_blocks_fallback) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -1428,53 +1397,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"fallback", nullptr}; \ - } \ + default: \ + return {"fallback", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"fallback", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -1490,145 +1461,143 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"fallback", nullptr}; \ - } \ + default: \ + return {"fallback", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"fallback", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ break - inline Kernel select_fallback_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"fallback", nullptr}; - } +inline Kernel select_fallback_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; } +} - inline Kernel select_fallback_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"fallback", nullptr}; - } +inline Kernel select_fallback_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"fallback", nullptr}; } +} - inline Kernel select_fallback_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"fallback", nullptr}; - } +inline Kernel select_fallback_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; } +} - inline Kernel select_fallback_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"fallback", nullptr}; - } +inline Kernel select_fallback_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"fallback", nullptr}; } +} - inline Kernel select_fallback_scalar_decompress_block_f32(const u8 bit_width, - const u32 block_size, - const bool sign_values) { - return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); - } +inline Kernel select_fallback_scalar_decompress_block_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +} - inline Kernel select_fallback_scalar_decompress_blocks_f32(const u8 bit_width, - const u32 block_size, - const bool sign_values) { - return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); - } +inline Kernel select_fallback_scalar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +} - inline Kernel select_fallback_scalar_decompress_block_f64(const u8 bit_width, - const u32 block_size, - const bool sign_values) { - return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); - } +inline Kernel select_fallback_scalar_decompress_block_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +} - inline Kernel select_fallback_scalar_decompress_blocks_f64(const u8 bit_width, - const u32 block_size, - const bool sign_values) { - return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); - } +inline Kernel select_fallback_scalar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +} - #define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) - #define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) - #define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) - #define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) - #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(12, BS); \ @@ -1644,53 +1613,55 @@ case N: \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"fallback_stdpar", nullptr}; \ - } \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } \ break - #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"fallback_stdpar", nullptr}; \ - } \ +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } \ break - #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(12, BS); \ @@ -1706,156 +1677,158 @@ case N: \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"fallback_stdpar", nullptr}; \ - } \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } \ break - #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"fallback_stdpar", nullptr}; \ - } \ +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } \ break - inline Kernel select_fallback_stdpar_decompress_block_f32(const u8 bit_width, const u32 block_size, - const bool sign_values) { +inline Kernel select_fallback_stdpar_decompress_block_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - switch (block_size) { - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(1024); - default: return {"fallback_stdpar", nullptr}; - } + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_stdpar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - const bool sign_values) { +inline Kernel select_fallback_stdpar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - switch (block_size) { - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(1024); - default: return {"fallback_stdpar", nullptr}; - } + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_stdpar_decompress_block_f64(const u8 bit_width, const u32 block_size, - const bool sign_values) { +inline Kernel select_fallback_stdpar_decompress_block_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - switch (block_size) { - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(1024); - default: return {"fallback_stdpar", nullptr}; - } + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_stdpar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - const bool sign_values) { +inline Kernel select_fallback_stdpar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - switch (block_size) { - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(1024); - default: return {"fallback_stdpar", nullptr}; - } + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_simd_decompress_block_f32(const u8, const u32, - const bool sign_values) { - static_cast(sign_values); +inline Kernel select_fallback_simd_decompress_block_f32(const u8, const u32, const bool sign_values) { + static_cast(sign_values); #if defined(PERNIX_BUILD_FALLBACK_SIMD) - if (sign_values) { - return {"fallback_simd", &decompress_block_fallback_simd<1, true, 64, f32>}; - } - return {"fallback_simd", &decompress_block_fallback_simd<1, false, 64, f32>}; + if (sign_values) { + return {"fallback_simd", &decompress_block_fallback_simd<1, true, 64, f32>}; + } + return {"fallback_simd", &decompress_block_fallback_simd<1, false, 64, f32>}; #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_simd_decompress_blocks_f32(const u8, const u32, - const bool sign_values) { - static_cast(sign_values); +inline Kernel select_fallback_simd_decompress_blocks_f32(const u8, const u32, const bool sign_values) { + static_cast(sign_values); #if defined(PERNIX_BUILD_FALLBACK_SIMD) - if (sign_values) { - return {"fallback_simd", &decompress_blocks_fallback_simd<1, true, 64, f32>}; - } - return {"fallback_simd", &decompress_blocks_fallback_simd<1, false, 64, f32>}; + if (sign_values) { + return {"fallback_simd", &decompress_blocks_fallback_simd<1, true, 64, f32>}; + } + return {"fallback_simd", &decompress_blocks_fallback_simd<1, false, 64, f32>}; #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_simd_decompress_block_f64(const u8, const u32, - const bool sign_values) { - static_cast(sign_values); +inline Kernel select_fallback_simd_decompress_block_f64(const u8, const u32, const bool sign_values) { + static_cast(sign_values); #if defined(PERNIX_BUILD_FALLBACK_SIMD) - if (sign_values) { - return {"fallback_simd", &decompress_block_fallback_simd<1, true, 64, f64>}; - } - return {"fallback_simd", &decompress_block_fallback_simd<1, false, 64, f64>}; + if (sign_values) { + return {"fallback_simd", &decompress_block_fallback_simd<1, true, 64, f64>}; + } + return {"fallback_simd", &decompress_block_fallback_simd<1, false, 64, f64>}; #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} - inline Kernel select_fallback_simd_decompress_blocks_f64(const u8, const u32, - const bool sign_values) { - static_cast(sign_values); +inline Kernel select_fallback_simd_decompress_blocks_f64(const u8, const u32, const bool sign_values) { + static_cast(sign_values); #if defined(PERNIX_BUILD_FALLBACK_SIMD) - if (sign_values) { - return {"fallback_simd", &decompress_blocks_fallback_simd<1, true, 64, f64>}; - } - return {"fallback_simd", &decompress_blocks_fallback_simd<1, false, 64, f64>}; + if (sign_values) { + return {"fallback_simd", &decompress_blocks_fallback_simd<1, true, 64, f64>}; + } + return {"fallback_simd", &decompress_blocks_fallback_simd<1, false, 64, f64>}; #else - return {"unsupported_implementation", nullptr}; + return {"unsupported_implementation", nullptr}; #endif - } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -1865,34 +1838,38 @@ case N: \ #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal #if defined(PERNIX_BUILD_X86_AVX2) namespace pernix::internal { #define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ -case N: return Kernel("avx2", &mm256_compress_block_avx2) + case N: \ + return Kernel("avx2", &mm256_compress_block_avx2) #define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ -case N: return Kernel("avx2", &mm256_compress_blocks_avx2) + case N: \ + return Kernel("avx2", &mm256_compress_blocks_avx2) #define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ -case N: return Kernel("avx2", &mm256_compress_block_avx2) + case N: \ + return Kernel("avx2", &mm256_compress_block_avx2) #define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ -case N: return Kernel("avx2", &mm256_compress_blocks_avx2) - -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + case N: \ + return Kernel("avx2", &mm256_compress_blocks_avx2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ @@ -1908,51 +1885,53 @@ case N: return Kernel("avx2", &mm256_compress_blocks_avx2("avx2", &mm256_compress_blocks_avx2 select_avx2_compress_block_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"avx2", nullptr}; - } - } - - inline Kernel select_avx2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"avx2", nullptr}; - } - } - - inline Kernel select_avx2_compress_block_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"avx2", nullptr}; - } - } - - inline Kernel select_avx2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"avx2", nullptr}; - } + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } + +inline Kernel select_avx2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +inline Kernel select_avx2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +inline Kernel select_avx2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } +} + +inline Kernel select_avx2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; } +} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 @@ -2065,41 +2046,40 @@ case N: return Kernel("avx2", &mm256_compress_blocks_avx2("avx2", &mm256_decompress_block_avx2); \ - return Kernel("avx2", &mm256_decompress_block_avx2) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ - return Kernel("avx2", &mm256_decompress_blocks_avx2) - - -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ - return Kernel("avx2", &mm256_decompress_block_avx2) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ - return Kernel("avx2", &mm256_decompress_blocks_avx2) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -2115,53 +2095,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"avx2", nullptr}; \ - } \ + default: \ + return {"avx2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"avx2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -2177,88 +2159,90 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"avx2", nullptr}; \ - } \ + default: \ + return {"avx2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"avx2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ break - inline Kernel select_avx2_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"avx2", nullptr}; - } +inline Kernel select_avx2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; } +} - inline Kernel select_avx2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"avx2", nullptr}; - } +inline Kernel select_avx2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"avx2", nullptr}; } +} - inline Kernel select_avx2_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"avx2", nullptr}; - } +inline Kernel select_avx2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; } +} - inline Kernel select_avx2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"avx2", nullptr}; - } +inline Kernel select_avx2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"avx2", nullptr}; } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -2276,35 +2260,39 @@ case N: \ #undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32 #undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64 -} +} // namespace pernix::internal #endif #if defined(PERNIX_BUILD_X86_BMI2) namespace pernix::internal { #define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ -case N: return Kernel("bmi2", &mm256_compress_block_bmi2) + case N: \ + return Kernel("bmi2", &mm256_compress_block_bmi2) #define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ -case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2) + case N: \ + return Kernel("bmi2", &mm256_compress_blocks_bmi2) #define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ -case N: return Kernel("bmi2", &mm256_compress_block_bmi2) + case N: \ + return Kernel("bmi2", &mm256_compress_block_bmi2) #define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ -case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2) - -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + case N: \ + return Kernel("bmi2", &mm256_compress_blocks_bmi2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ @@ -2320,51 +2308,53 @@ case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2("bmi2", &mm256_compress_blocks_bmi2 select_bmi2_compress_block_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"bmi2", nullptr}; - } - } - - inline Kernel select_bmi2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"bmi2", nullptr}; - } - } - - inline Kernel select_bmi2_compress_block_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"bmi2", nullptr}; - } - } - - inline Kernel select_bmi2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"bmi2", nullptr}; - } + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } + +inline Kernel select_bmi2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +inline Kernel select_bmi2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +inline Kernel select_bmi2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +inline Kernel select_bmi2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; } +} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 @@ -2469,41 +2461,40 @@ case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2("bmi2", &mm256_decompress_block_bmi2); \ - return Kernel("bmi2", &mm256_decompress_block_bmi2) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ - return Kernel("bmi2", &mm256_decompress_blocks_bmi2) - - -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ - return Kernel("bmi2", &mm256_decompress_block_bmi2) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ - return Kernel("bmi2", &mm256_decompress_blocks_bmi2) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -2519,53 +2510,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"bmi2", nullptr}; \ - } \ + default: \ + return {"bmi2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"bmi2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -2581,88 +2574,90 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"bmi2", nullptr}; \ - } \ + default: \ + return {"bmi2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"bmi2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ break - inline Kernel select_bmi2_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"bmi2", nullptr}; - } +inline Kernel select_bmi2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; } +} - inline Kernel select_bmi2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"bmi2", nullptr}; - } +inline Kernel select_bmi2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"bmi2", nullptr}; } +} - inline Kernel select_bmi2_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"bmi2", nullptr}; - } +inline Kernel select_bmi2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; } +} - inline Kernel select_bmi2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"bmi2", nullptr}; - } +inline Kernel select_bmi2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"bmi2", nullptr}; } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -2672,35 +2667,39 @@ case N: \ #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) namespace pernix::internal { #define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ -case N: return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + case N: \ + return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) #define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ -case N: return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + case N: \ + return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) #define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ -case N: return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + case N: \ + return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) #define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ -case N: return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) - -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + case N: \ + return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ @@ -2716,51 +2715,53 @@ case N: return Kernel("avx512vbmi", &mm512_compress_blocks_ PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ + default: \ + return {"avx512vbmi", nullptr}; \ } #define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } - -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ @@ -2776,86 +2777,88 @@ case N: return Kernel("avx512vbmi", &mm512_compress_blocks_ PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ + default: \ + return {"avx512vbmi", nullptr}; \ } #define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } - - inline Kernel select_avx512vbmi_compress_block_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"avx512vbmi", nullptr}; - } - } - - inline Kernel select_avx512vbmi_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"avx512vbmi", nullptr}; - } - } - - inline Kernel select_avx512vbmi_compress_block_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"avx512vbmi", nullptr}; - } - } - - inline Kernel select_avx512vbmi_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"avx512vbmi", nullptr}; - } + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +inline Kernel select_avx512vbmi_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +inline Kernel select_avx512vbmi_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +inline Kernel select_avx512vbmi_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; } +} + +inline Kernel select_avx512vbmi_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 @@ -2865,41 +2868,40 @@ case N: return Kernel("avx512vbmi", &mm512_compress_blocks_ #undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 #undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 -} +} // namespace pernix::internal namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ - return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ - return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) - - -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ - return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ - return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -2915,53 +2917,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -2977,88 +2981,90 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ break - inline Kernel select_avx512vbmi_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"avx512vbmi", nullptr}; - } +inline Kernel select_avx512vbmi_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; } +} - inline Kernel select_avx512vbmi_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"avx512vbmi", nullptr}; - } +inline Kernel select_avx512vbmi_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"avx512vbmi", nullptr}; } +} - inline Kernel select_avx512vbmi_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"avx512vbmi", nullptr}; - } +inline Kernel select_avx512vbmi_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; } +} - inline Kernel select_avx512vbmi_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"avx512vbmi", nullptr}; - } +inline Kernel select_avx512vbmi_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"avx512vbmi", nullptr}; } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -3068,72 +3074,72 @@ case N: \ #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal #endif #if defined(PERNIX_BUILD_ARM64_NEON) namespace pernix::internal { - inline Kernel select_neon_compress_block_f32(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"neon", nullptr}; - } +inline Kernel select_neon_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} - inline Kernel select_neon_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"neon", nullptr}; - } +inline Kernel select_neon_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} - inline Kernel select_neon_compress_block_f64(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"neon", nullptr}; - } +inline Kernel select_neon_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} - inline Kernel select_neon_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"neon", nullptr}; - } +inline Kernel select_neon_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; } +} // namespace pernix::internal using pernix::arm64::neon::neon_decompress_block; using pernix::arm64::neon::neon_decompress_blocks; namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_block); \ - return Kernel("neon", &neon_decompress_block) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ - return Kernel("neon", &neon_decompress_blocks) - -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_block); \ - return Kernel("neon", &neon_decompress_block) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ - return Kernel("neon", &neon_decompress_blocks) - -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -3149,53 +3155,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"neon", nullptr}; \ - } \ + default: \ + return {"neon", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"neon", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -3211,39 +3219,41 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"neon", nullptr}; \ - } \ + default: \ + return {"neon", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"neon", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ break inline Kernel select_neon_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { @@ -3302,72 +3312,72 @@ inline Kernel select_neon_decompress_blocks_f64(const u8 bi #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal #endif #if defined(PERNIX_BUILD_ARM64_SVE2) namespace pernix::internal { - inline Kernel select_sve2_compress_block_f32(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"sve2", nullptr}; - } +inline Kernel select_sve2_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} - inline Kernel select_sve2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"sve2", nullptr}; - } +inline Kernel select_sve2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} - inline Kernel select_sve2_compress_block_f64(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"sve2", nullptr}; - } +inline Kernel select_sve2_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} - inline Kernel select_sve2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"sve2", nullptr}; - } +inline Kernel select_sve2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; } +} // namespace pernix::internal using pernix::arm64::sve2::sve2_decompress_block; using pernix::arm64::sve2::sve2_decompress_blocks; namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ - return Kernel("sve2", &sve2_decompress_block) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ - return Kernel("sve2", &sve2_decompress_blocks) - -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ - return Kernel("sve2", &sve2_decompress_block) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ - return Kernel("sve2", &sve2_decompress_blocks) - -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -3383,53 +3393,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"sve2", nullptr}; \ - } \ + default: \ + return {"sve2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"sve2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -3445,88 +3457,90 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"sve2", nullptr}; \ - } \ + default: \ + return {"sve2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"sve2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ break - inline Kernel select_sve2_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"sve2", nullptr}; - } +inline Kernel select_sve2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"sve2", nullptr}; } +} - inline Kernel select_sve2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"sve2", nullptr}; - } +inline Kernel select_sve2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"sve2", nullptr}; } +} - inline Kernel select_sve2_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"sve2", nullptr}; - } +inline Kernel select_sve2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"sve2", nullptr}; } +} - inline Kernel select_sve2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"sve2", nullptr}; - } +inline Kernel select_sve2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"sve2", nullptr}; } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -3536,7 +3550,7 @@ case N: \ #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal #endif -#endif //PERNIX_SELECT_IMPL_H +#endif // PERNIX_SELECT_IMPL_H diff --git a/include/pernix/fallback/common.h b/include/pernix/fallback/common.h index 2e15530..46d44d2 100644 --- a/include/pernix/fallback/common.h +++ b/include/pernix/fallback/common.h @@ -3,39 +3,50 @@ #include +#include +#include +#include + namespace pernix::internal { -__always_inline f32 dequantize_epi32_ps(const i32 input, const f32 scale) { - return static_cast(input) * scale; -} -__always_inline f64 dequantize_epi32_pd(const i64 input, const f64 scale) { - return static_cast(input) * scale; -} +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && std::is_integral_v) +__always_inline T quantize_clamped(const FloatT input, const FloatT scale) { + constexpr T min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + + constexpr T max_value = BIT_WIDTH == 1 ? 1 : (1 << (BIT_WIDTH - 1)) - 1; + + const FloatT scaled = input * scale; -__always_inline i32 quantize_ps_epi32(const f32 input, const f32 scale) { - return static_cast(std::lroundf(input * scale)); + if (std::isnan(scaled)) [[unlikely]] { + return 0; + } + + const FloatT clamped = std::clamp(scaled, static_cast(min_value), static_cast(max_value)); + + return static_cast(std::lrint(clamped)); } -__always_inline i64 quantize_pd_epi64(const f64 input, const f64 scale) { - return std::llround(input * scale); +template + requires(std::is_integral_v && std::is_floating_point_v) +__always_inline FloatT dequantize(const T input, const FloatT scale) { + return static_cast(input) * scale; } -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 32) -__always_inline i32 sign_extend(const u32 value) { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__always_inline constexpr i32 sign_extend(const u32 value) noexcept { if constexpr (BIT_WIDTH == 1) { - return static_cast(value & 1U); + return static_cast(value & 1); } constexpr u32 sign_bit = 1 << (BIT_WIDTH - 1); - constexpr u32 mask = (1 << BIT_WIDTH) - 1U; + constexpr u32 mask = (1 << BIT_WIDTH) - 1; const u32 masked = value & mask; - return static_cast( - static_cast(masked ^ sign_bit) - static_cast(sign_bit) - ); -} + return static_cast(static_cast(masked ^ sign_bit) - static_cast(sign_bit)); } +} // namespace pernix::internal -#endif //PERNIX_FALLBACK_COMMON_H +#endif // PERNIX_FALLBACK_COMMON_H diff --git a/include/pernix/fallback/scalar_compression.h b/include/pernix/fallback/scalar_compression.h index 9c30324..cbbe968 100644 --- a/include/pernix/fallback/scalar_compression.h +++ b/include/pernix/fallback/scalar_compression.h @@ -2,134 +2,39 @@ #define PERNIX_FALLBACK_SCALAR_COMPRESSION_H #include +#include #include #include -#include #include #include #include -#include namespace pernix { namespace internal { -__always_inline i32 quantize_ps_epi32(const float input, const float scale) { - return static_cast(std::lroundf(input * scale)); -} - -__always_inline i64 quantize_pd_epi64(const f64 input, const f64 scale) { - return std::llround(input * scale); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -__always_inline i32 quantize_clamped(const T input, const T scale) { - constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); - constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); - - const long double scaled = static_cast(input) * static_cast(scale); - if (std::isnan(scaled)) { - return 0; - } - if (scaled <= static_cast(min_value)) { - return static_cast(min_value); - } - if (scaled >= static_cast(max_value)) { - return static_cast(max_value); - } - - return static_cast(std::llround(scaled)); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline i32 clamp_signed_quantized(const i64 value) { - if constexpr (BIT_WIDTH == 1) { - return static_cast(std::clamp(value, 0, 1)); - } - - constexpr i32 min_value = -(1 << (BIT_WIDTH - 1)); - constexpr i32 max_value = (1 << (BIT_WIDTH - 1)) - 1; - return static_cast(std::clamp(value, min_value, max_value)); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) -void pack_epi32_fallback_inner(const std::span input, const u8 bit_offset, - u8* __restrict__ destination) { - constexpr u32 bits_in_type = sizeof(T) * 8; - constexpr u32 bitmask = BIT_WIDTH == bits_in_type - ? std::numeric_limits::max() - : (1U << BIT_WIDTH) - 1U; - - std::size_t idx = 0; - std::size_t bits_in_buffer = bit_offset; - u64 buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; - -#pragma GCC unroll 64 - for (u32 raw_value : input) { - const u32 next_value = raw_value & bitmask; - - buffer |= static_cast(next_value) << bits_in_buffer; - bits_in_buffer += BIT_WIDTH; - - while (bits_in_buffer >= 8) { - destination[idx++] = static_cast(buffer & 0xFFU); - buffer >>= 8; - bits_in_buffer -= 8; - } - } - - if (bits_in_buffer > 0) { - destination[idx] = static_cast(buffer & 0xFFU); - } -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -void pack_epi32_fallback(const std::span input, u8* __restrict__ destination) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } else { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -void pack_epi32_fallback(const std::vector& input, u8* __restrict__ destination) { - return pack_epi32_fallback(std::span(input.data(), input.size()), destination); -} - template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v && - std::is_floating_point_v) -void quantize_and_pack_fallback_inner(const FloatT* __restrict__ input, const FloatT scale, - const u32 elements, const u8 bit_offset, - u8* __restrict__ destination) { + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v && std::is_floating_point_v) +__always_inline void quantize_and_pack_fallback_inner(const std::span input, const FloatT scale, const u32 elements, + const u8 bit_offset, std::span destination) { constexpr u32 bits_in_type = sizeof(T) * 8; - constexpr u32 bitmask = BIT_WIDTH == bits_in_type - ? std::numeric_limits::max() - : (1U << BIT_WIDTH) - 1U; + constexpr T bitmask = BIT_WIDTH == bits_in_type ? std::numeric_limits::max() : static_cast((1U << BIT_WIDTH) - 1U); - std::size_t idx = 0; - std::size_t bits_in_buffer = bit_offset; - u64 buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; + usize idx = 0; + usize bits_in_buffer = bit_offset; + u64 buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; -#pragma GCC unroll 64 - for (u32 i = 0; i < elements; i++) { - const i32 quantized = quantize_clamped(input[i], scale); - const u32 next_value = static_cast(quantized) & bitmask; +#pragma GCC unroll 512 + for (usize i = 0; i < elements; i++) { + const i32 quantized = quantize_clamped(input[i], scale); + const u32 next_value = static_cast(quantized) & static_cast(bitmask); - buffer |= static_cast(next_value) << bits_in_buffer; + buffer |= static_cast(next_value) << bits_in_buffer; bits_in_buffer += BIT_WIDTH; while (bits_in_buffer >= 8) { destination[idx++] = static_cast(buffer & 0xFFU); - buffer >>= 8; - bits_in_buffer -= 8; + buffer >>= 8; + bits_in_buffer -= 8; } } @@ -140,85 +45,64 @@ void quantize_and_pack_fallback_inner(const FloatT* __restrict__ input, const Fl template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -void quantize_and_pack_fallback(const FloatT* __restrict__ input, const FloatT scale, - const u32 elements, u8* __restrict__ destination) { +__always_inline void quantize_and_pack_fallback(const std::span input, const FloatT scale, const u32 elements, + std::span destination) { if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); + quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); + quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); } else { - return quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); + quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); } } -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_block_fallback(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); +} // namespace internal +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_block_fallback(const std::span input, const FloatT scale, std::span destination) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - std::memset(output, 0, BLOCK_SIZE); + std::ranges::fill(destination, 0); - internal::quantize_and_pack_fallback(input, scale, elements_per_block, output); + internal::quantize_and_pack_fallback(input, scale, elements_per_block, destination); return 0; } -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_block_fallback(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_block_fallback(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - std::memset(output, 0, BLOCK_SIZE); + const auto input_span = std::span(static_cast(input_ptr), elements_per_block); + auto output_span = std::span(static_cast(output_ptr), BLOCK_SIZE); - internal::quantize_and_pack_fallback(input, scale, elements_per_block, output); - return 0; + return compress_block_fallback(input_span, scale, output_span); } -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_blocks_fallback(const void* __restrict__ input_ptr, float scale, void* __restrict__ output_ptr, - u32 blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const f32* block_input = input; - u8* block_output = output; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_blocks_fallback(const std::span input, const FloatT scale, std::span destination, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - for (u32 block = 0; block < blocks; block++) { + for (usize i = 0; i < blocks; i++) { + const auto block_input = input.subspan(i * elements_per_block, elements_per_block); + auto block_output = destination.subspan(i * BLOCK_SIZE, BLOCK_SIZE); compress_block_fallback(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; } return 0; } -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_blocks_fallback(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr, - const unsigned int blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_blocks_fallback(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const f64* block_input = input; - u8* block_output = output; + const auto input_span = std::span(static_cast(input_ptr), elements_per_block * blocks); + auto output_span = std::span(static_cast(output_ptr), blocks * BLOCK_SIZE); - for (u32 block = 0; block < blocks; block++) { - compress_block_fallback(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - return 0; -} + return compress_blocks_fallback(input_span, scale, output_span, blocks); } +} // namespace pernix -#endif // PERNIX_FALLBACK_SCALAR_COMPRESSION_H +#endif // PERNIX_FALLBACK_SCALAR_COMPRESSION_H diff --git a/include/pernix/fallback/scalar_decompression.h b/include/pernix/fallback/scalar_decompression.h index fc7bad0..126cb40 100644 --- a/include/pernix/fallback/scalar_decompression.h +++ b/include/pernix/fallback/scalar_decompression.h @@ -2,105 +2,31 @@ #define PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H #include +#include #include +#include #include -#include namespace pernix { namespace internal { -__always_inline float dequantize_epi32(const i32 input, const float scale) { - return static_cast(input) * scale; -} - -__always_inline f64 dequantize_epi64(const i64 input, const f64 scale) { - return static_cast(input) * scale; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline auto sign_extend(const u32 value) -> i32 { - if constexpr (BIT_WIDTH == 1) { - return static_cast(value & 1U); - } - - constexpr u32 sign_bit = u32{1} << (BIT_WIDTH - 1); - constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1; - const u32 masked = value & mask; - return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) -__always_inline std::vector unpack_epi32_fallback_inner(const u8* __restrict__ input, const u8 bit_offset, - const std::size_t elements) { - constexpr u32 bits_in_type = sizeof(T) * 8; - constexpr u32 bitmask = BIT_WIDTH == bits_in_type - ? std::numeric_limits::max() - : (1U << BIT_WIDTH) - 1U; - - std::vector output(elements); - - std::size_t idx = 0; - u8 bits_in_buffer = 8 - bit_offset; - u64 buffer = static_cast(input[idx++]) >> bit_offset; - -#pragma GCC unroll 64 - for (u32 i = 0; i < elements; i++) { - while (BIT_WIDTH > bits_in_buffer) { - const auto next_value = static_cast(input[idx++]) << bits_in_buffer; - buffer |= next_value; - bits_in_buffer += 8; - } - - const u32 raw_value = static_cast(buffer & bitmask); - if constexpr (SIGN_VALUES) { - output[i] = sign_extend(raw_value); - } else { - output[i] = static_cast(raw_value); - } - - buffer >>= BIT_WIDTH; - bits_in_buffer -= BIT_WIDTH; - } - - return output; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline auto unpack_epi32_fallback(const u8* __restrict__ input, - const std::size_t elements) -> std::vector { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else { - return unpack_epi32_fallback_inner(input, 0, elements); - } -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v && - std::is_floating_point_v) -__always_inline void unpack_and_dequantize_fallback_inner(const u8* __restrict__ input, const u8 bit_offset, - const u32 elements, const ScaleType scale, - ScaleType* __restrict__ output) { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v && std::is_floating_point_v) +__always_inline void unpack_and_dequantize_fallback_inner(const std::span input, const u8 bit_offset, const u32 elements, + const FloatT scale, std::span output) { constexpr u32 bits_in_type = sizeof(T) * 8; - constexpr u32 bitmask = BIT_WIDTH == bits_in_type - ? std::numeric_limits::max() - : (1U << BIT_WIDTH) - 1U; + constexpr T bitmask = BIT_WIDTH == bits_in_type ? std::numeric_limits::max() : static_cast((1U << BIT_WIDTH) - 1U); - std::size_t idx = 0; + usize idx = 0; u8 bits_in_buffer = 8 - bit_offset; u64 buffer = static_cast(input[idx++]) >> bit_offset; -#pragma GCC unroll 64 - for (u32 i = 0; i < elements; i++) { +#pragma GCC unroll 512 + for (usize i = 0; i < elements; i++) { while (BIT_WIDTH > bits_in_buffer) { const auto next_value = static_cast(input[idx++]) << bits_in_buffer; - buffer |= next_value; - bits_in_buffer += 8; + buffer |= next_value; + bits_in_buffer += 8; } const u32 raw_value = static_cast(buffer & bitmask); @@ -110,90 +36,73 @@ __always_inline void unpack_and_dequantize_fallback_inner(const u8* __restrict__ } else { unpacked = static_cast(raw_value); } - output[i] = static_cast(unpacked) * scale; + output[i] = static_cast(unpacked) * scale; - buffer >>= BIT_WIDTH; + buffer >>= BIT_WIDTH; bits_in_buffer -= BIT_WIDTH; } } -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -__always_inline void unpack_and_dequantize_fallback(const u8* __restrict__ input, const u32 elements, - const ScaleType scale, ScaleType* __restrict__ output) { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +__always_inline void unpack_and_dequantize_fallback(const std::span input, const u32 elements, const FloatT scale, + std::span output) { if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); + unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); + unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); } else { - return unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); + unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); } } -} - -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_block_fallback(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); +} // namespace internal +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_block_fallback(const std::span input, const FloatT scale, std::span output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; internal::unpack_and_dequantize_fallback(input, elements_per_block, scale, output); return 0; } -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_block_fallback(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_block_fallback(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - internal::unpack_and_dequantize_fallback(input, elements_per_block, scale, output); - return 0; -} + const auto input = std::span(static_cast(input_ptr), BLOCK_SIZE); + auto output = std::span(static_cast(output_ptr), elements_per_block); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int decompress_blocks_fallback(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr, const u32 blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + return decompress_block_fallback(input, scale, output); +} - const u8* block_input = input; - f32* block_output = output; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0 && std::is_floating_point_v) +int decompress_blocks_fallback(const std::span input, const FloatT scale, std::span output, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; for (u32 block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + const auto block_input = input.subspan(block * BLOCK_SIZE, BLOCK_SIZE); + auto block_output = output.subspan(block * elements_per_block, elements_per_block); + + decompress_block_fallback(block_input, scale, block_output); } return 0; } -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_blocks_fallback(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr, const u32 blocks) { - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); - - const u8* block_input = input; - f64* block_output = output; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0 && std::is_floating_point_v) +int decompress_blocks_fallback(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - for (u32 block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } + const auto input_span = std::span(static_cast(input_ptr), blocks * BLOCK_SIZE); + auto output_span = std::span(static_cast(output_ptr), elements_per_block * blocks); - return 0; -} + return decompress_blocks_fallback(input_span, scale, output_span, blocks); } -#endif // PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H +} // namespace pernix + +#endif // PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H diff --git a/src/fallback/fallback_compression.cpp b/src/fallback/fallback_compression.cpp index 021ac27..1e3a075 100644 --- a/src/fallback/fallback_compression.cpp +++ b/src/fallback/fallback_compression.cpp @@ -1,32 +1,36 @@ #include -#include #include +#include namespace pernix::internal { #define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ -case N: return Kernel("fallback", &compress_block_fallback) + case N: \ + return Kernel("fallback", &compress_block_fallback) #define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ -case N: return Kernel("fallback", &compress_blocks_fallback) + case N: \ + return Kernel("fallback", &compress_blocks_fallback) #define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ -case N: return Kernel("fallback", &compress_block_fallback) + case N: \ + return Kernel("fallback", &compress_block_fallback) #define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ -case N: return Kernel("fallback", &compress_blocks_fallback) + case N: \ + return Kernel("fallback", &compress_blocks_fallback) -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ @@ -42,51 +46,53 @@ case N: return Kernel("fallback", &compress_blocks_fallback PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ - default: return {"fallback", nullptr}; \ + default: \ + return {"fallback", nullptr}; \ } #define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ - default: return {"fallback", nullptr}; \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ } -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ @@ -102,86 +108,88 @@ case N: return Kernel("fallback", &compress_blocks_fallback PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ - default: return {"fallback", nullptr}; \ + default: \ + return {"fallback", nullptr}; \ } #define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ - default: return {"fallback", nullptr}; \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ } - Kernel select_fallback_compress_block_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"fallback", nullptr}; - } +Kernel select_fallback_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; } +} - Kernel select_fallback_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"fallback", nullptr}; - } +Kernel select_fallback_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; } +} - Kernel select_fallback_compress_block_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"fallback", nullptr}; - } +Kernel select_fallback_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; } +} - Kernel select_fallback_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"fallback", nullptr}; - } +Kernel select_fallback_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; } +} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 @@ -191,4 +199,4 @@ case N: return Kernel("fallback", &compress_blocks_fallback #undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 #undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 -} +} // namespace pernix::internal diff --git a/src/fallback/fallback_decompression.cpp b/src/fallback/fallback_decompression.cpp index 72d5866..44e8db0 100644 --- a/src/fallback/fallback_decompression.cpp +++ b/src/fallback/fallback_decompression.cpp @@ -2,38 +2,38 @@ #include namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ - return Kernel("fallback", &decompress_block_fallback) +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ - return Kernel("fallback", &decompress_blocks_fallback) +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ - return Kernel("fallback", &decompress_block_fallback) +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ - return Kernel("fallback", &decompress_blocks_fallback) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -49,53 +49,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"fallback", nullptr}; \ - } \ + default: \ + return {"fallback", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"fallback", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -111,88 +113,90 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"fallback", nullptr}; \ - } \ + default: \ + return {"fallback", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"fallback", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ break - Kernel select_fallback_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"fallback", nullptr}; - } +Kernel select_fallback_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; } +} - Kernel select_fallback_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"fallback", nullptr}; - } +Kernel select_fallback_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"fallback", nullptr}; } +} - Kernel select_fallback_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"fallback", nullptr}; - } +Kernel select_fallback_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; } +} - Kernel select_fallback_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"fallback", nullptr}; - } +Kernel select_fallback_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"fallback", nullptr}; } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -202,4 +206,4 @@ case N: \ #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal From fe14648e49f81aa6000e635f520cf3d2e1ebf882 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 9 Jul 2026 22:40:27 +0200 Subject: [PATCH 32/43] style: apply consistent formatting and spacing across header files and implementation files --- examples/c/roundtrip.c | 15 +- examples/cpp/roundtrip.cpp | 15 +- include/pernix/arm64/neon/common.h | 123 +- include/pernix/arm64/neon/compression.h | 40 +- include/pernix/arm64/neon/decompression.h | 88 +- include/pernix/arm64/neon/packing.h | 3 +- include/pernix/arm64/neon/tables.h | 26 +- include/pernix/arm64/neon/unpacking.h | 5 +- include/pernix/arm64/sve2/compression.h | 17 +- include/pernix/arm64/sve2/decompression.h | 93 +- include/pernix/arm64/sve2/packing.h | 6 +- include/pernix/arm64/sve2/tables.h | 10 +- include/pernix/arm64/sve2/unpacking.h | 10 +- include/pernix/backend.hpp | 18 +- include/pernix/compat.h | 2 +- include/pernix/detail/api.hpp | 70 +- include/pernix/dispatch/cpu_features.h | 26 +- include/pernix/dispatch/kernel.h | 12 +- include/pernix/dispatch/select.h | 212 ++-- include/pernix/fallback/simd_compression.h | 12 +- include/pernix/fallback/simd_decompression.h | 14 +- include/pernix/fallback/stdpar_common.h | 39 +- include/pernix/fallback/stdpar_compression.h | 29 +- .../pernix/fallback/stdpar_decompression.h | 35 +- include/pernix/pernix.h | 58 +- include/pernix/pernix.hpp | 152 +-- include/pernix/simd_compat.h | 1 + include/pernix/x86/avx2/avx2_compression.h | 304 +++-- include/pernix/x86/avx2/avx2_decompression.h | 192 ++- include/pernix/x86/avx2/avx2_tables.h | 6 +- .../x86/avx512vbmi/avx512vbmi_compression.h | 352 +++--- .../x86/avx512vbmi/avx512vbmi_decompression.h | 281 ++--- include/pernix/x86/avx512vbmi/compat.h | 6 +- include/pernix/x86/avx512vbmi/packing.h | 86 +- include/pernix/x86/avx512vbmi/tables.h | 23 +- include/pernix/x86/avx512vbmi/unpacking.h | 11 +- include/pernix/x86/bmi2/bmi2_compression.h | 171 ++- include/pernix/x86/bmi2/bmi2_decompression.h | 203 ++- include/pernix/x86/utils.h | 2 +- src/arm64/neon/compression.cpp | 42 +- src/arm64/neon/decompression.cpp | 208 ++-- src/arm64/sve2/compression.cpp | 42 +- src/arm64/sve2/decompression.cpp | 288 ++--- src/dispatch/cpu_features_arm.cpp | 6 +- src/dispatch/cpu_features_x86.cpp | 15 +- src/dispatch/select.cpp | 1105 ++++++++--------- src/pernix.cpp | 78 +- src/x86/avx2/avx2_compression.cpp | 258 ++-- src/x86/avx2/avx2_decompression.cpp | 287 ++--- src/x86/avx512vbmi/avx512vbmi_compression.cpp | 258 ++-- .../avx512vbmi/avx512vbmi_decompression.cpp | 287 ++--- src/x86/bmi2/bmi2_compression.cpp | 258 ++-- src/x86/bmi2/bmi2_decompression.cpp | 287 ++--- tests/c_api_roundtrip.c | 12 +- tests/c_api_validation.c | 92 +- tests/deterministic_correctness_tests.cpp | 235 ++-- tests/fallback_tests.cpp | 323 +++-- tests/header_only_tests.cpp | 70 +- tests/include/testset.h | 134 +- tests/install_consumer/main.c | 7 +- tests/install_consumer/main.cpp | 13 +- tests/simd_tests.cpp | 64 +- 62 files changed, 3282 insertions(+), 3855 deletions(-) diff --git a/examples/c/roundtrip.c b/examples/c/roundtrip.c index ad64824..e1660b5 100644 --- a/examples/c/roundtrip.c +++ b/examples/c/roundtrip.c @@ -13,29 +13,28 @@ int main(void) { } for (int i = 0; i < elements; ++i) { - input[i] = ((float)i - 16.0f) * 0.125f; + input[i] = ((float)i - 16.0f) * 0.125f; const float magnitude = input[i] < 0.0f ? -input[i] : input[i]; - bmax = bmax < magnitude ? magnitude : bmax; + bmax = bmax < magnitude ? magnitude : bmax; } - float scale = 0.0f; + float scale = 0.0f; float inverse_scale = 0.0f; if (pernix_decompression_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK || pernix_inverse_scale_f32(scale, &inverse_scale) != PERNIX_STATUS_OK) { return 2; } - if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, inverse_scale, - compressed) != PERNIX_STATUS_OK) { + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { return 3; } - if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, - true) != PERNIX_STATUS_OK) { + if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, true) != + PERNIX_STATUS_OK) { return 4; } for (int i = 0; i < elements; ++i) { - const float diff = restored[i] - input[i]; + const float diff = restored[i] - input[i]; const float error = diff < 0.0f ? -diff : diff; if (error > scale) { return 5; diff --git a/examples/cpp/roundtrip.cpp b/examples/cpp/roundtrip.cpp index f2381f7..34ffcff 100644 --- a/examples/cpp/roundtrip.cpp +++ b/examples/cpp/roundtrip.cpp @@ -1,13 +1,12 @@ -#include - #include #include #include #include +#include int main() { - constexpr u8 bit_width = 16; - constexpr u32 block_size = pernix::compressed_block_size(); + constexpr u8 bit_width = 16; + constexpr u32 block_size = pernix::compressed_block_size(); constexpr std::size_t elements = pernix::elements_per_block(bit_width); std::array input{}; @@ -19,7 +18,7 @@ int main() { for (const float value : input) { bmax = std::max(bmax, std::abs(value)); } - float scale = 0.0f; + float scale = 0.0f; float inverse_scale = 0.0f; if (pernix::decompression_scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK || pernix::inverse_scale(scale, inverse_scale) != PERNIX_STATUS_OK) { @@ -29,12 +28,10 @@ int main() { std::array compressed{}; std::array restored{}; - if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, inverse_scale, compressed) != - PERNIX_STATUS_OK) { + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { return 1; } - if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, scale, restored) != - PERNIX_STATUS_OK) { + if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, scale, restored) != PERNIX_STATUS_OK) { return 2; } diff --git a/include/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h index cda7065..dcfd664 100644 --- a/include/pernix/arm64/neon/common.h +++ b/include/pernix/arm64/neon/common.h @@ -20,43 +20,35 @@ __always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(const int8x16_t& inpu const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); - return { - { - vmovl_s16(vget_low_s16(s16_lo)), - vmovl_s16(vget_high_s16(s16_lo)), - vmovl_s16(vget_low_s16(s16_hi)), - vmovl_s16(vget_high_s16(s16_hi)), - } - }; + return {{ + vmovl_s16(vget_low_s16(s16_lo)), + vmovl_s16(vget_high_s16(s16_lo)), + vmovl_s16(vget_low_s16(s16_hi)), + vmovl_s16(vget_high_s16(s16_hi)), + }}; } __always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(const int16x8_t& input) { - return { - { - vmovl_s16(vget_low_s16(input)), - vmovl_s16(vget_high_s16(input)), - } - }; + return {{ + vmovl_s16(vget_low_s16(input)), + vmovl_s16(vget_high_s16(input)), + }}; } __always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, const float32x4_t& scale) { - return { - { - vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), - } - }; + return {{ + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), + }}; } __always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t& input, const float32x4_t& scale) { - return { - { - vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), - } - }; + return {{ + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + }}; } __always_inline float32x4_t neon_dequantize_epi32(const int32x4_t& input, const float32x4_t& scale) { @@ -68,26 +60,22 @@ __always_inline float64x2_t neon_dequantize_epi32_f64(const int32x2_t& input, co } __always_inline float64x2x2_t neon_dequantize_epi32_f64(const int32x4_t& input, const float64x2_t& scale) { - return { - { - neon_dequantize_epi32_f64(vget_low_s32(input), scale), - neon_dequantize_epi32_f64(vget_high_s32(input), scale), - } - }; + return {{ + neon_dequantize_epi32_f64(vget_low_s32(input), scale), + neon_dequantize_epi32_f64(vget_high_s32(input), scale), + }}; } __always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t& input, const float64x2_t& scale) { const float64x2x2_t dequantized_low = neon_dequantize_epi32_f64(input.val[0], scale); const float64x2x2_t dequantized_high = neon_dequantize_epi32_f64(input.val[1], scale); - return { - { - dequantized_low.val[0], - dequantized_low.val[1], - dequantized_high.val[0], - dequantized_high.val[1], - } - }; + return {{ + dequantized_low.val[0], + dequantized_low.val[1], + dequantized_high.val[0], + dequantized_high.val[1], + }}; } __always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input, const float64x2_t& scale) { @@ -96,18 +84,16 @@ __always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input const float64x2x2_t dequantized2 = neon_dequantize_epi32_f64(input.val[2], scale); const float64x2x2_t dequantized3 = neon_dequantize_epi32_f64(input.val[3], scale); - return { - { - dequantized0.val[0], - dequantized0.val[1], - dequantized1.val[0], - dequantized1.val[1], - dequantized2.val[0], - dequantized2.val[1], - dequantized3.val[0], - dequantized3.val[1], - } - }; + return {{ + dequantized0.val[0], + dequantized0.val[1], + dequantized1.val[0], + dequantized1.val[1], + dequantized2.val[0], + dequantized2.val[1], + dequantized3.val[0], + dequantized3.val[1], + }}; } __always_inline uint8x16_t neon_load_tail_elements_int8(const u8* input, const u32 tail_bytes_count) { @@ -140,8 +126,7 @@ __always_inline float64x2_t neon_load_tail_elements_f64(const u8* input, const u return vld1q_f64(buffer); } -__always_inline void neon_store_tail_elements_int8(u8* output, const uint8x16x4_t& data, - const u32 tail_elements) { +__always_inline void neon_store_tail_elements_int8(u8* output, const uint8x16x4_t& data, const u32 tail_elements) { u8 buffer[16 * 4]; for (u32 i = 0; i < 4; ++i) { vst1q_u8(buffer + i * 16, data.val[i]); @@ -149,8 +134,7 @@ __always_inline void neon_store_tail_elements_int8(u8* output, const uint8x16x4_ std::memcpy(output, buffer, tail_elements * sizeof(u8)); } -__always_inline void neon_store_tail_elements_int16(u16* output, const uint16x8x4_t& data, - const u32 tail_elements) { +__always_inline void neon_store_tail_elements_int16(u16* output, const uint16x8x4_t& data, const u32 tail_elements) { u16 buffer[8 * 4]; for (u32 i = 0; i < 4; ++i) { vst1q_u16(buffer + i * 8, data.val[i]); @@ -158,8 +142,7 @@ __always_inline void neon_store_tail_elements_int16(u16* output, const uint16x8x std::memcpy(output, buffer, tail_elements * sizeof(u16)); } -__always_inline void neon_store_tail_elements_int32(u32* output, const uint32x4x4_t& data, - const u32 tail_elements) { +__always_inline void neon_store_tail_elements_int32(u32* output, const uint32x4x4_t& data, const u32 tail_elements) { u32 buffer[4 * 4]; for (u32 i = 0; i < 4; ++i) { vst1q_u32(buffer + i * 4, data.val[i]); @@ -167,8 +150,7 @@ __always_inline void neon_store_tail_elements_int32(u32* output, const uint32x4x std::memcpy(output, buffer, tail_elements * sizeof(u32)); } -__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x4_t& data, - const u32 tail_elements) { +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x4_t& data, const u32 tail_elements) { float32_t buffer[16 * 4]; for (u32 i = 0; i < 4; ++i) { vst1q_f32(buffer + i * 4, data.val[i]); @@ -176,8 +158,7 @@ __always_inline void neon_store_tail_elements_f32(float32_t* output, const float std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); } -__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x2_t& data, - const u32 tail_elements) { +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x2_t& data, const u32 tail_elements) { float32_t buffer[8 * 2]; for (u32 i = 0; i < 2; ++i) { vst1q_f32(buffer + i * 4, data.val[i]); @@ -185,15 +166,13 @@ __always_inline void neon_store_tail_elements_f32(float32_t* output, const float std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); } -__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4_t& data, - const u32 tail_elements) { +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4_t& data, const u32 tail_elements) { float32_t buffer[4]; vst1q_f32(buffer, data); std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); } -__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x4_t& data, - const u32 tail_elements) { +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x4_t& data, const u32 tail_elements) { float64_t buffer[2 * 4]; for (u32 i = 0; i < 4; ++i) { vst1q_f64(buffer + i * 2, data.val[i]); @@ -201,8 +180,7 @@ __always_inline void neon_store_tail_elements_f64(float64_t* output, const float std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); } -__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x2_t& data, - const u32 tail_elements) { +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x2_t& data, const u32 tail_elements) { float64_t buffer[2 * 2]; for (u32 i = 0; i < 2; ++i) { vst1q_f64(buffer + i * 2, data.val[i]); @@ -210,14 +188,13 @@ __always_inline void neon_store_tail_elements_f64(float64_t* output, const float std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); } -__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x8_t& data, - const u32 tail_elements) { +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x8_t& data, const u32 tail_elements) { float64_t buffer[2 * 8]; for (u32 i = 0; i < 8; ++i) { vst1q_f64(buffer + i * 2, data.val[i]); } std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); } -} // namespace pernix::arm64::neon::internal +} // namespace pernix::arm64::neon::internal #endif // PERNIX_ARM64_NEON_COMMON_H diff --git a/include/pernix/arm64/neon/compression.h b/include/pernix/arm64/neon/compression.h index 3a03199..63adcb2 100644 --- a/include/pernix/arm64/neon/compression.h +++ b/include/pernix/arm64/neon/compression.h @@ -1,8 +1,8 @@ #ifndef PERNIX_ARM64_NEON_COMPRESSION_H #define PERNIX_ARM64_NEON_COMPRESSION_H -#include #include +#include #include #include @@ -11,57 +11,50 @@ namespace pernix::arm64::neon { namespace internal { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_1to8(const u8* __restrict__ input, const f32 scale, - f32* __restrict__ output) { +__always_inline int neon_compress_block_1to8(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { static_assert(true, "Not yet implemented"); return -1; } template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_9to16(const u8* __restrict__ input, const f32 scale, - f32* __restrict__ output) { +__always_inline int neon_compress_block_9to16(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { static_assert(true, "Not yet implemented"); return -1; } template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_17to24(const u8* __restrict__ input, const f32 scale, - f32* __restrict__ output) { +__always_inline int neon_compress_block_17to24(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { static_assert(true, "Not yet implemented"); return -1; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_1to8(const u8* __restrict__ input, const f64 scale, - f64* __restrict__ output) { +__always_inline int neon_compress_block_1to8(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { static_assert(true, "Not yet implemented"); return -1; } template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_9to16(const u8* __restrict__ input, const f64 scale, - f64* __restrict__ output) { +__always_inline int neon_compress_block_9to16(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { static_assert(true, "Not yet implemented"); return -1; } template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block_17to24(const u8* __restrict__ input, const f64 scale, - f64* __restrict__ output) { +__always_inline int neon_compress_block_17to24(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { static_assert(true, "Not yet implemented"); return -1; } -} // namespace internal +} // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block(const u8* __restrict__ input, const f32 scale, - f32* __restrict__ output) { +__always_inline int neon_compress_block(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return internal::neon_compress_block_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { @@ -74,8 +67,7 @@ __always_inline int neon_compress_block(const u8* __restrict__ input, const f32 template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_compress_block(const u8* __restrict__ input, const f64 scale, - f64* __restrict__ output) { +__always_inline int neon_compress_block(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return internal::neon_compress_block_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { @@ -88,14 +80,13 @@ __always_inline int neon_compress_block(const u8* __restrict__ input, const f64 template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_compress_blocks(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output, - const u32 blocks) { +int neon_compress_blocks(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output, const u32 blocks) { const u8* block_input = input; f32* block_output = output; for (u32 block = 0; block < blocks; ++block) { neon_compress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -104,18 +95,17 @@ int neon_compress_blocks(const u8* __restrict__ input, const f32 scale, f32* __r template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_compress_blocks(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output, - const u32 blocks) { +int neon_compress_blocks(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output, const u32 blocks) { const u8* block_input = input; f64* block_output = output; for (u32 block = 0; block < blocks; ++block) { neon_compress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix::arm64::neon +} // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_COMPRESSION_H diff --git a/include/pernix/arm64/neon/decompression.h b/include/pernix/arm64/neon/decompression.h index 375adc2..f688f6a 100644 --- a/include/pernix/arm64/neon/decompression.h +++ b/include/pernix/arm64/neon/decompression.h @@ -12,8 +12,7 @@ namespace pernix::arm64::neon { namespace internal { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_16 = elements_per_block / 16; @@ -23,9 +22,7 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input for (uint32_t i = 0; i < iterations_16; ++i) { const uint8x16_t source = vld1q_u8(input); - const int8x16_t unpacked = b128::neon_unpack_epi8_1to8 - (source); + const int8x16_t unpacked = b128::neon_unpack_epi8_1to8(source); const int32x4x4_t converted = neon_convert_int8x16_int32x4x4(unpacked); const float32x4x4_t dequantized = neon_dequantize_epi32(converted, scale_v); @@ -39,11 +36,8 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input } if constexpr (remaining_elements > 0) { - const uint8x16_t tail_source = neon_load_tail_elements_int8( - input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8 - (tail_source); + const uint8x16_t tail_source = neon_load_tail_elements_int8(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8(tail_source); const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x4(tail_unpacked); const float32x4x4_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); @@ -56,8 +50,7 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; @@ -67,9 +60,7 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu for (uint32_t i = 0; i < iterations_8; ++i) { const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); - const int16x8_t unpacked = b128::neon_unpack_epi16_9to16 - (source); + const int16x8_t unpacked = b128::neon_unpack_epi16_9to16(source); const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); const float32x4x2_t dequantized = neon_dequantize_epi32(converted, scale_v); @@ -83,11 +74,8 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu } if constexpr (remaining_elements > 0) { - const uint16x8_t tail_source = neon_load_tail_elements_int16( - input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16 - (tail_source); + const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16(tail_source); const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); const float32x4x2_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); @@ -100,8 +88,7 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_4 = elements_per_block / 4; @@ -144,8 +131,7 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp if constexpr (tail_bit_offset == 0) { tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } else { - tail_unpacked = b128::neon_unpack_epi32_17to24( - tail_source); + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } const float32x4_t tail_dequantized = neon_dequantize_epi32(tail_unpacked, scale_v); @@ -158,8 +144,7 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_16 = elements_per_block / 16; @@ -169,9 +154,7 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input for (uint32_t i = 0; i < iterations_16; ++i) { const uint8x16_t source = vld1q_u8(input); - const int8x16_t unpacked = b128::neon_unpack_epi8_1to8 - (source); + const int8x16_t unpacked = b128::neon_unpack_epi8_1to8(source); const int32x4x4_t converted = neon_convert_int8x16_int32x4x4(unpacked); const float64x2x8_t dequantized = neon_dequantize_epi32_f64(converted, scale_v); @@ -185,11 +168,8 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input } if constexpr (remaining_elements > 0) { - const uint8x16_t tail_source = neon_load_tail_elements_int8( - input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8 - (tail_source); + const uint8x16_t tail_source = neon_load_tail_elements_int8(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8(tail_source); const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x4(tail_unpacked); const float64x2x8_t tail_dequantized = neon_dequantize_epi32_f64(tail_converted, scale_v); @@ -202,8 +182,7 @@ __always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_8 = elements_per_block / 8; @@ -213,9 +192,7 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu for (uint32_t i = 0; i < iterations_8; ++i) { const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); - const int16x8_t unpacked = b128::neon_unpack_epi16_9to16 - (source); + const int16x8_t unpacked = b128::neon_unpack_epi16_9to16(source); const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); const float64x2x4_t dequantized = neon_dequantize_epi32_f64(converted, scale_v); @@ -229,11 +206,8 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu } if constexpr (remaining_elements > 0) { - const uint16x8_t tail_source = neon_load_tail_elements_int16( - input, tail_bytes(BIT_WIDTH, remaining_elements)); - const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16 - (tail_source); + const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16(tail_source); const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); const float64x2x4_t tail_dequantized = neon_dequantize_epi32_f64(tail_converted, scale_v); @@ -246,8 +220,7 @@ __always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ inpu template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr uint32_t iterations_4 = elements_per_block / 4; @@ -291,8 +264,7 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp if constexpr (tail_bit_offset == 0) { tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } else { - tail_unpacked = b128::neon_unpack_epi32_17to24( - tail_source); + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); } const float64x2x2_t tail_dequantized = neon_dequantize_epi32_f64(tail_unpacked, scale_v); @@ -302,12 +274,11 @@ __always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ inp return 0; } -} // namespace internal +} // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block(const void* __restrict__ input, const float_t scale, - void* __restrict__ output) { +__always_inline int neon_decompress_block(const void* __restrict__ input, const float_t scale, void* __restrict__ output) { const auto* typed_input = static_cast(input); auto* typed_output = static_cast(output); if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { @@ -322,8 +293,7 @@ __always_inline int neon_decompress_block(const void* __restrict__ input, const template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int neon_decompress_block(const void* __restrict__ input, const double_t scale, - void* __restrict__ output) { +__always_inline int neon_decompress_block(const void* __restrict__ input, const double_t scale, void* __restrict__ output) { const auto* typed_input = static_cast(input); auto* typed_output = static_cast(output); if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { @@ -338,8 +308,7 @@ __always_inline int neon_decompress_block(const void* __restrict__ input, const template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_blocks(const void* __restrict__ input, const float_t scale, void* __restrict__ output, - const uint32_t blocks) { +int neon_decompress_blocks(const void* __restrict__ input, const float_t scale, void* __restrict__ output, const uint32_t blocks) { const auto* typed_input = static_cast(input); auto* typed_output = static_cast(output); const uint8_t* block_input = typed_input; @@ -347,7 +316,7 @@ int neon_decompress_blocks(const void* __restrict__ input, const float_t scale, for (uint32_t block = 0; block < blocks; ++block) { neon_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -356,8 +325,7 @@ int neon_decompress_blocks(const void* __restrict__ input, const float_t scale, template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int neon_decompress_blocks(const void* __restrict__ input, const double_t scale, void* __restrict__ output, - const uint32_t blocks) { +int neon_decompress_blocks(const void* __restrict__ input, const double_t scale, void* __restrict__ output, const uint32_t blocks) { const auto* typed_input = static_cast(input); auto* typed_output = static_cast(output); const uint8_t* block_input = typed_input; @@ -365,12 +333,12 @@ int neon_decompress_blocks(const void* __restrict__ input, const double_t scale, for (uint32_t block = 0; block < blocks; ++block) { neon_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix::arm64::neon +} // namespace pernix::arm64::neon #endif // PERNIX_ARM64_NEON_DECOMPRESSION_H diff --git a/include/pernix/arm64/neon/packing.h b/include/pernix/arm64/neon/packing.h index 538b5a8..a972dcf 100644 --- a/include/pernix/arm64/neon/packing.h +++ b/include/pernix/arm64/neon/packing.h @@ -3,7 +3,6 @@ #include -namespace pernix::arm64::neon::internal { -} // namespace pernix::arm64::neon::internal +namespace pernix::arm64::neon::internal {} // namespace pernix::arm64::neon::internal #endif // PERNIX_ARM64_NEON_PACKING_H diff --git a/include/pernix/arm64/neon/tables.h b/include/pernix/arm64/neon/tables.h index 2cf4e37..dad2453 100644 --- a/include/pernix/arm64/neon/tables.h +++ b/include/pernix/arm64/neon/tables.h @@ -13,9 +13,7 @@ inline constexpr u8 inactive_lane = 0xff; template constexpr bool table_indices_are_valid(const std::array& table) { - return std::ranges::all_of(table, [](const u8 index) { - return index == inactive_lane || index < Elements; - }); + return std::ranges::all_of(table, [](const u8 index) { return index == inactive_lane || index < Elements; }); } template @@ -131,7 +129,7 @@ constexpr std::array make_shift_right_32() { return table; } -} // namespace detail +} // namespace detail template struct table_unpacking; @@ -146,12 +144,9 @@ struct table_unpacking { public: static constexpr u8 bit_width = BIT_WIDTH; - alignas(64) static constexpr std::array permute1 = - detail::make_primary_permute(); - alignas(64) static constexpr std::array permute2 = - detail::make_spill_permute(); - alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); + alignas(64) static constexpr std::array permute1 = detail::make_primary_permute(); + alignas(64) static constexpr std::array permute2 = detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); alignas(64) static constexpr std::array shift2 = detail::make_shift_left_for_spill(); @@ -173,10 +168,8 @@ struct table_unpacking { alignas(64) static constexpr std::array permute1 = detail::make_primary_permute(); - alignas(64) static constexpr std::array permute2 = - detail::make_spill_permute(); - alignas(64) static constexpr std::array shift1 = - detail::make_shift_right(); + alignas(64) static constexpr std::array permute2 = detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); alignas(64) static constexpr std::array shift2 = detail::make_shift_left_for_spill(); @@ -187,8 +180,7 @@ struct table_unpacking { }; template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && VECTOR_WIDTH == detail::neon_vector_width && START_BIT_OFFSET < - 8) + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && VECTOR_WIDTH == detail::neon_vector_width && START_BIT_OFFSET < 8) struct table_unpacking { private: static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; @@ -209,6 +201,6 @@ struct table_unpacking { template struct table_packing; -} // namespace pernix::arm64::internal +} // namespace pernix::arm64::neon::internal #endif // PERNIX_ARM64_NEON_TABLES_H diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h index 1d227c1..2648676 100644 --- a/include/pernix/arm64/neon/unpacking.h +++ b/include/pernix/arm64/neon/unpacking.h @@ -60,8 +60,7 @@ __always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t& input) { if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { const uint8x16_t permuted2_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute2.data())); - const uint16x8_t shifted2 = vshlq_u16(vreinterpretq_u16_u8(permuted2_u8), - vld1q_s16(tables::shift2.data())); + const uint16x8_t shifted2 = vshlq_u16(vreinterpretq_u16_u8(permuted2_u8), vld1q_s16(tables::shift2.data())); shifted = vorrq_u16(shifted, shifted2); } @@ -96,6 +95,6 @@ __always_inline int32x4_t neon_unpack_epi32_17to24(const uint32x4_t& input) { return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); } } -} // namespace pernix::arm64::neon::internal::b128 +} // namespace pernix::arm64::neon::internal::b128 #endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/arm64/sve2/compression.h b/include/pernix/arm64/sve2/compression.h index e441f22..8ddc0b9 100644 --- a/include/pernix/arm64/sve2/compression.h +++ b/include/pernix/arm64/sve2/compression.h @@ -10,40 +10,35 @@ namespace pernix { namespace internal { template inline constexpr bool sve2_compression_unimplemented_v = false; -} // namespace internal - +} // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_compress_block(const f32*, f32, u8*) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); return -1; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_compress_block(const f64*, f64, u8*) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); return -1; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_compress_blocks(const f32*, f32, u8*, u32) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); return -1; } template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) int sve2_compress_blocks(const f64*, f64, u8*, u32) { - static_assert(internal::sve2_compression_unimplemented_v, - "ARM64 SVE2 compression is not implemented yet"); + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); return -1; } -} // namespace pernix +} // namespace pernix #endif // PERNIX_ARM64_SVE2_COMPRESSION_H diff --git a/include/pernix/arm64/sve2/decompression.h b/include/pernix/arm64/sve2/decompression.h index 350a196..3effd31 100644 --- a/include/pernix/arm64/sve2/decompression.h +++ b/include/pernix/arm64/sve2/decompression.h @@ -17,15 +17,13 @@ template return (elements * BIT_WIDTH + 7) / 8; } -[[nodiscard]] __always_inline svuint8_t sve2_load_packed_bytes(const uint8_t* __restrict__ input, - const uint32_t bytes) { +[[nodiscard]] __always_inline svuint8_t sve2_load_packed_bytes(const uint8_t* __restrict__ input, const uint32_t bytes) { const svbool_t pg = svwhilelt_b8(uint64_t{0}, static_cast(bytes)); return svld1_u8(pg, input); } template -__always_inline void sve2_store_dequantized_i8_f32(svint8_t values, const svfloat32_t scale_v, - float_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i8_f32(svint8_t values, const svfloat32_t scale_v, float_t* __restrict__ output, const uint32_t count) { alignas(64) std::vector temp(svcntb()); @@ -52,8 +50,7 @@ __always_inline void sve2_store_dequantized_i8_f32(svint8_t values, const svfloa } template -__always_inline void sve2_store_dequantized_i8_f64(svint8_t values, const double_t scale, - double_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i8_f64(svint8_t values, const double_t scale, double_t* __restrict__ output, const uint32_t count) { std::vector temp(svcntb()); @@ -69,8 +66,7 @@ __always_inline void sve2_store_dequantized_i8_f64(svint8_t values, const double } template -__always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfloat32_t scale_v, - float_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfloat32_t scale_v, float_t* __restrict__ output, const uint32_t count) { alignas(64) std::vector temp(svcnth()); @@ -85,9 +81,8 @@ __always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfl const svint32_t widened = svld1sh_s32(pg, temp.data() + offset); dequantized = svmul_f32_x(pg, svcvt_f32_s32_x(pg, widened), scale_v); } else { - const svuint32_t widened = - svld1uh_u32(pg, reinterpret_cast(temp.data() + offset)); - dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, widened), scale_v); + const svuint32_t widened = svld1uh_u32(pg, reinterpret_cast(temp.data() + offset)); + dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, widened), scale_v); } svst1_f32(pg, output + offset, dequantized); @@ -97,8 +92,7 @@ __always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfl } template -__always_inline void sve2_store_dequantized_i16_f64(svint16_t values, const double_t scale, - double_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i16_f64(svint16_t values, const double_t scale, double_t* __restrict__ output, const uint32_t count) { std::vector temp(svcnth()); @@ -114,8 +108,7 @@ __always_inline void sve2_store_dequantized_i16_f64(svint16_t values, const doub } template -__always_inline void sve2_store_dequantized_i32_f32(svint32_t values, const svfloat32_t scale_v, - float_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i32_f32(svint32_t values, const svfloat32_t scale_v, float_t* __restrict__ output, const uint32_t count) { const svbool_t pg = svwhilelt_b32(uint64_t{0}, static_cast(count)); @@ -130,8 +123,7 @@ __always_inline void sve2_store_dequantized_i32_f32(svint32_t values, const svfl } template -__always_inline void sve2_store_dequantized_i32_f64(svint32_t values, const double_t scale, - double_t* __restrict__ output, +__always_inline void sve2_store_dequantized_i32_f64(svint32_t values, const double_t scale, double_t* __restrict__ output, const uint32_t count) { std::vector temp(svcntw()); @@ -148,8 +140,7 @@ __always_inline void sve2_store_dequantized_i32_f64(svint32_t values, const doub template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcntb()); @@ -175,14 +166,12 @@ __always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input const uint8_t* chunk_input = input + input_bit_offset / 8; const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); - const svint8_t unpacked = sve2_unpack_epi8_1to8 - (source, permute, shift, spill_permute, spill_shift); + const svint8_t unpacked = sve2_unpack_epi8_1to8(source, permute, shift, spill_permute, spill_shift); sve2_store_dequantized_i8_f32(unpacked, scale_v, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; @@ -190,8 +179,7 @@ __always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcnth()); @@ -216,16 +204,14 @@ __always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ inpu const uint32_t bytes = packed_bytes(count); const uint8_t* chunk_input = input + input_bit_offset / 8; - const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); const svint16_t unpacked = - sve2_unpack_epi16_9to16 - (svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); + sve2_unpack_epi16_9to16(svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); sve2_store_dequantized_i16_f32(unpacked, scale_v, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; @@ -233,8 +219,7 @@ __always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ inpu template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcntw()); @@ -261,7 +246,7 @@ __always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ inp sve2_store_dequantized_i32_f32(unpacked, scale_v, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; @@ -269,8 +254,7 @@ __always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ inp template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcntb()); @@ -294,14 +278,12 @@ __always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input const uint8_t* chunk_input = input + input_bit_offset / 8; const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); - const svint8_t unpacked = sve2_unpack_epi8_1to8 - (source, permute, shift, spill_permute, spill_shift); + const svint8_t unpacked = sve2_unpack_epi8_1to8(source, permute, shift, spill_permute, spill_shift); sve2_store_dequantized_i8_f64(unpacked, scale, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; @@ -309,8 +291,7 @@ __always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcnth()); @@ -333,16 +314,14 @@ __always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ inpu const uint32_t bytes = packed_bytes(count); const uint8_t* chunk_input = input + input_bit_offset / 8; - const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); const svint16_t unpacked = - sve2_unpack_epi16_9to16 - (svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); + sve2_unpack_epi16_9to16(svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); sve2_store_dequantized_i16_f64(unpacked, scale, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; @@ -350,8 +329,7 @@ __always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ inpu template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto lanes = static_cast(svcntw()); @@ -376,12 +354,12 @@ __always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ inp sve2_store_dequantized_i32_f64(unpacked, scale, output + processed_elements, count); processed_elements += count; - input_bit_offset += count * BIT_WIDTH; + input_bit_offset += count * BIT_WIDTH; } return 0; } -} // namespace internal +} // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) @@ -399,8 +377,7 @@ int sve2_decompress_block(const void* __restrict__ input, const float_t scale, v template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_block(const void* __restrict__ input, const double_t scale, - void* __restrict__ output) { +int sve2_decompress_block(const void* __restrict__ input, const double_t scale, void* __restrict__ output) { const auto* typed_input = static_cast(input); auto* typed_output = static_cast(output); if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { @@ -414,8 +391,7 @@ int sve2_decompress_block(const void* __restrict__ input, const double_t scale, template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_blocks(const void* __restrict__ input, const float_t scale, void* __restrict__ output, - const uint32_t blocks) { +int sve2_decompress_blocks(const void* __restrict__ input, const float_t scale, void* __restrict__ output, const uint32_t blocks) { const auto* typed_input = static_cast(input); auto* typed_output = static_cast(output); const uint8_t* block_input = typed_input; @@ -423,7 +399,7 @@ int sve2_decompress_blocks(const void* __restrict__ input, const float_t scale, for (uint32_t block = 0; block < blocks; ++block) { sve2_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -432,8 +408,7 @@ int sve2_decompress_blocks(const void* __restrict__ input, const float_t scale, template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int sve2_decompress_blocks(const void* __restrict__ input, const double_t scale, void* __restrict__ output, - const uint32_t blocks) { +int sve2_decompress_blocks(const void* __restrict__ input, const double_t scale, void* __restrict__ output, const uint32_t blocks) { const auto* typed_input = static_cast(input); auto* typed_output = static_cast(output); const uint8_t* block_input = typed_input; @@ -441,12 +416,12 @@ int sve2_decompress_blocks(const void* __restrict__ input, const double_t scale, for (uint32_t block = 0; block < blocks; ++block) { sve2_decompress_block(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix::arm64::sve2 +} // namespace pernix::arm64::sve2 #endif // PERNIX_ARM64_SVE2_DECOMPRESSION_H diff --git a/include/pernix/arm64/sve2/packing.h b/include/pernix/arm64/sve2/packing.h index 7d644f2..802c612 100644 --- a/include/pernix/arm64/sve2/packing.h +++ b/include/pernix/arm64/sve2/packing.h @@ -4,8 +4,8 @@ #include namespace pernix::arm64::sve2::internal { - template - inline constexpr bool packing_unimplemented_v = false; -} // namespace pernix::arm64::sve2::internal +template +inline constexpr bool packing_unimplemented_v = false; +} // namespace pernix::arm64::sve2::internal #endif // PERNIX_ARM64_SVE2_PACKING_H diff --git a/include/pernix/arm64/sve2/tables.h b/include/pernix/arm64/sve2/tables.h index a11fcde..e6b0a79 100644 --- a/include/pernix/arm64/sve2/tables.h +++ b/include/pernix/arm64/sve2/tables.h @@ -98,9 +98,8 @@ struct table_unpacking { if constexpr (BIT_WIDTH % 8u != 0) { constexpr u8 extra_bits = BIT_WIDTH % 8u; const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); - const svuint8_t low_bits = - svadd_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), START_BIT_OFFSET); - first = svadd_u8_x(pg, first, svadd_u8_x(pg, high, svlsr_n_u8_x(pg, low_bits, 3))); + const svuint8_t low_bits = svadd_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), START_BIT_OFFSET); + first = svadd_u8_x(pg, first, svadd_u8_x(pg, high, svlsr_n_u8_x(pg, low_bits, 3))); } return svadd_u8_x(pg, first, byte); @@ -108,10 +107,9 @@ struct table_unpacking { static svuint32_t shift() { const svbool_t pg = svptrue_b32(); - return svand_n_u32_x(pg, svadd_n_u32_x(pg, svmul_n_u32_x(pg, svindex_u32(0, 1), BIT_WIDTH), - START_BIT_OFFSET), 7); + return svand_n_u32_x(pg, svadd_n_u32_x(pg, svmul_n_u32_x(pg, svindex_u32(0, 1), BIT_WIDTH), START_BIT_OFFSET), 7); } }; -} // namespace pernix::arm64::sve2::internal +} // namespace pernix::arm64::sve2::internal #endif // PERNIX_ARM64_SVE2_TABLES_H diff --git a/include/pernix/arm64/sve2/unpacking.h b/include/pernix/arm64/sve2/unpacking.h index 3d0825a..824a2b6 100644 --- a/include/pernix/arm64/sve2/unpacking.h +++ b/include/pernix/arm64/sve2/unpacking.h @@ -43,8 +43,7 @@ __always_inline svint8_t sve2_unpack_epi8_1to8(const svuint8_t input, const svui template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline svint16_t sve2_unpack_epi16_9to16(const svuint16_t input, const svuint8_t permute, - const svuint16_t shift, +__always_inline svint16_t sve2_unpack_epi16_9to16(const svuint16_t input, const svuint8_t permute, const svuint16_t shift, const svuint8_t spill_permute, const svuint16_t spill_shift) { if constexpr (BIT_WIDTH == 16) { return svreinterpret_s16(input); @@ -56,9 +55,8 @@ __always_inline svint16_t sve2_unpack_epi16_9to16(const svuint16_t input, const if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { const svuint8_t spill_permuted_values = svtbl_u8(svreinterpret_u8_u16(input), spill_permute); - const svuint16_t spill_shifted = svlsl_u16_x(pg, svreinterpret_u16_u8(spill_permuted_values), - spill_shift); - shifted = svorr_u16_x(pg, shifted, spill_shifted); + const svuint16_t spill_shifted = svlsl_u16_x(pg, svreinterpret_u16_u8(spill_permuted_values), spill_shift); + shifted = svorr_u16_x(pg, shifted, spill_shifted); } constexpr int sign_shift = 16 - BIT_WIDTH; @@ -89,6 +87,6 @@ __always_inline svint32_t sve2_unpack_epi32_17to24(const svuint8_t input) { return svreinterpret_s32_u32(svand_n_u32_x(pg, unpacked, mask)); } } -} // namespace pernix::arm64::sve2::internal +} // namespace pernix::arm64::sve2::internal #endif // PERNIX_ARM64_SVE2_UNPACKING_H diff --git a/include/pernix/backend.hpp b/include/pernix/backend.hpp index aa1ea32..7e9d4bc 100644 --- a/include/pernix/backend.hpp +++ b/include/pernix/backend.hpp @@ -5,17 +5,17 @@ namespace pernix { enum class Backend { - Auto = PERNIX_BACKEND_AUTO, - Fallback = PERNIX_BACKEND_FALLBACK, + Auto = PERNIX_BACKEND_AUTO, + Fallback = PERNIX_BACKEND_FALLBACK, FallbackScalar = PERNIX_BACKEND_FALLBACK_SCALAR, - X86Avx2 = PERNIX_BACKEND_X86_AVX2, - X86Bmi2 = PERNIX_BACKEND_X86_BMI2, - X86Avx512Vbmi = PERNIX_BACKEND_X86_AVX512_VBMI, - Arm64Neon = PERNIX_BACKEND_ARM64_NEON, - Arm64Sve = PERNIX_BACKEND_ARM64_SVE, + X86Avx2 = PERNIX_BACKEND_X86_AVX2, + X86Bmi2 = PERNIX_BACKEND_X86_BMI2, + X86Avx512Vbmi = PERNIX_BACKEND_X86_AVX512_VBMI, + Arm64Neon = PERNIX_BACKEND_ARM64_NEON, + Arm64Sve = PERNIX_BACKEND_ARM64_SVE, FallbackStdpar = PERNIX_BACKEND_FALLBACK_STDPAR, - FallbackSimd = PERNIX_BACKEND_FALLBACK_SIMD + FallbackSimd = PERNIX_BACKEND_FALLBACK_SIMD }; } -#endif //PERNIX_BACKEND_HPP +#endif // PERNIX_BACKEND_HPP diff --git a/include/pernix/compat.h b/include/pernix/compat.h index b749f87..c81a396 100644 --- a/include/pernix/compat.h +++ b/include/pernix/compat.h @@ -46,4 +46,4 @@ typedef double f64; typedef size_t usize; -#endif //PERNIX_COMPAT_H +#endif // PERNIX_COMPAT_H diff --git a/include/pernix/detail/api.hpp b/include/pernix/detail/api.hpp index b540d41..53aef32 100644 --- a/include/pernix/detail/api.hpp +++ b/include/pernix/detail/api.hpp @@ -1,17 +1,17 @@ #ifndef PERNIX_DETAIL_API_HPP #define PERNIX_DETAIL_API_HPP -#include #include #include #include +#include #include #include namespace pernix::detail { -constexpr u8 min_bit_width = 1; -constexpr u8 max_bit_width = 24; +constexpr u8 min_bit_width = 1; +constexpr u8 max_bit_width = 24; constexpr u32 fixed_block_size = 64; constexpr bool is_valid_bit_width(const u8 bit_width) { @@ -19,8 +19,7 @@ constexpr bool is_valid_bit_width(const u8 bit_width) { } constexpr bool is_valid_block_size(const u32 block_size) { - return block_size == fixed_block_size || block_size == 128 || block_size == 256 || block_size == 512 || - block_size == 1024; + return block_size == fixed_block_size || block_size == 128 || block_size == 256 || block_size == 512 || block_size == 1024; } constexpr u32 elements_per_block(const u8 bit_width, const u32 block_size = fixed_block_size) { @@ -33,7 +32,7 @@ constexpr ScaleType quantization_levels(const u8 bit_width) { } template -inline pernix_status scale_from_bmax(const ScaleType bmax, const u8 bit_width, ScaleType *scale) { +inline pernix_status scale_from_bmax(const ScaleType bmax, const u8 bit_width, ScaleType* scale) { if (scale == nullptr || !is_valid_bit_width(bit_width) || !std::isfinite(bmax) || bmax < ScaleType{0}) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -52,7 +51,7 @@ inline bool is_valid_scale(const ScaleType scale) { } template -inline pernix_status inverse_scale(const ScaleType scale, ScaleType *inverse_scale_value) { +inline pernix_status inverse_scale(const ScaleType scale, ScaleType* inverse_scale_value) { if (inverse_scale_value == nullptr || !is_valid_scale(scale)) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -62,9 +61,8 @@ inline pernix_status inverse_scale(const ScaleType scale, ScaleType *inverse_sca } template -inline pernix_status compression_scale_from_bmax(const ScaleType bmax, const u8 bit_width, - ScaleType *inverse_scale_value) { - ScaleType scale = ScaleType{0}; +inline pernix_status compression_scale_from_bmax(const ScaleType bmax, const u8 bit_width, ScaleType* inverse_scale_value) { + ScaleType scale = ScaleType{0}; const auto status = scale_from_bmax(bmax, bit_width, &scale); if (status != PERNIX_STATUS_OK) { return status; @@ -72,22 +70,22 @@ inline pernix_status compression_scale_from_bmax(const ScaleType bmax, const u8 return inverse_scale(scale, inverse_scale_value); } -inline const char *status_string(const pernix_status status) { +inline const char* status_string(const pernix_status status) { switch (status) { - case PERNIX_STATUS_OK: - return "PERNIX_STATUS_OK"; - case PERNIX_STATUS_INVALID_ARGUMENT: - return "PERNIX_STATUS_INVALID_ARGUMENT"; - case PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH: - return "PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH"; - case PERNIX_STATUS_UNSUPPORTED_BACKEND: - return "PERNIX_STATUS_UNSUPPORTED_BACKEND"; - case PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE: - return "PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE"; - case PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION: - return "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION"; - default: - return "PERNIX_STATUS_UNKNOWN"; + case PERNIX_STATUS_OK: + return "PERNIX_STATUS_OK"; + case PERNIX_STATUS_INVALID_ARGUMENT: + return "PERNIX_STATUS_INVALID_ARGUMENT"; + case PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH: + return "PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH"; + case PERNIX_STATUS_UNSUPPORTED_BACKEND: + return "PERNIX_STATUS_UNSUPPORTED_BACKEND"; + case PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE: + return "PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE"; + case PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION: + return "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION"; + default: + return "PERNIX_STATUS_UNKNOWN"; } } @@ -102,8 +100,8 @@ inline pernix_status select_error_status(const std::string_view kernel_name) { } template -inline pernix_status compress_block(const Backend backend, const u8 bit_width, const u32 block_size, - const void *input, const ScaleType scale, void *output) { +inline pernix_status compress_block(const Backend backend, const u8 bit_width, const u32 block_size, const void* input, + const ScaleType scale, void* output) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -129,8 +127,8 @@ inline pernix_status compress_block(const Backend backend, const u8 bit_width, c } template -inline pernix_status compress_blocks(const Backend backend, const u8 bit_width, const u32 block_size, - const void *input, const ScaleType scale, void *output, const u32 blocks) { +inline pernix_status compress_blocks(const Backend backend, const u8 bit_width, const u32 block_size, const void* input, + const ScaleType scale, void* output, const u32 blocks) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -159,9 +157,8 @@ inline pernix_status compress_blocks(const Backend backend, const u8 bit_width, } template -inline pernix_status decompress_block(const Backend backend, const u8 bit_width, const u32 block_size, - const void *input, const ScaleType scale, void *output, - const bool sign_values) { +inline pernix_status decompress_block(const Backend backend, const u8 bit_width, const u32 block_size, const void* input, + const ScaleType scale, void* output, const bool sign_values) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -187,9 +184,8 @@ inline pernix_status decompress_block(const Backend backend, const u8 bit_width, } template -inline pernix_status decompress_blocks(const Backend backend, const u8 bit_width, const u32 block_size, - const void *input, const ScaleType scale, void *output, const u32 blocks, - const bool sign_values) { +inline pernix_status decompress_blocks(const Backend backend, const u8 bit_width, const u32 block_size, const void* input, + const ScaleType scale, void* output, const u32 blocks, const bool sign_values) { if (input == nullptr || output == nullptr) { return PERNIX_STATUS_INVALID_ARGUMENT; } @@ -216,6 +212,6 @@ inline pernix_status decompress_blocks(const Backend backend, const u8 bit_width return static_cast(kernel.func(input, scale, output, blocks)); } -} +} // namespace pernix::detail -#endif //PERNIX_DETAIL_API_HPP +#endif // PERNIX_DETAIL_API_HPP diff --git a/include/pernix/dispatch/cpu_features.h b/include/pernix/dispatch/cpu_features.h index c25b7c9..fb797e4 100644 --- a/include/pernix/dispatch/cpu_features.h +++ b/include/pernix/dispatch/cpu_features.h @@ -48,9 +48,7 @@ inline CpuFeatures detect_cpu_features() { #if defined(_MSC_VER) __cpuidex(regs, 1, 0); - const auto xgetbv = [](const unsigned int index) -> u64 { - return _xgetbv(index); - }; + const auto xgetbv = [](const unsigned int index) -> u64 { return _xgetbv(index); }; #else __cpuid_count(1, 0, regs[0], regs[1], regs[2], regs[3]); const auto xgetbv = [](const unsigned int index) -> u64 { @@ -62,12 +60,12 @@ inline CpuFeatures detect_cpu_features() { #endif const bool osxsave = (regs[2] & (1 << 27)) != 0; - const bool avx = (regs[2] & (1 << 28)) != 0; + const bool avx = (regs[2] & (1 << 28)) != 0; if (!osxsave || !avx) { return features; } - const u64 xcr0 = xgetbv(0); + const u64 xcr0 = xgetbv(0); const bool xmm_enabled = (xcr0 & 0x2) != 0; const bool ymm_enabled = (xcr0 & 0x4) != 0; const bool zmm_enabled = (xcr0 & 0x20) != 0 && (xcr0 & 0x40) != 0 && (xcr0 & 0x80) != 0; @@ -84,10 +82,10 @@ inline CpuFeatures detect_cpu_features() { features.avx2 = (regs[1] & (1 << 5)) != 0; features.bmi2 = (regs[1] & (1 << 8)) != 0; if (zmm_enabled) { - features.avx512f = (regs[1] & (1 << 16)) != 0; - features.avx512dq = (regs[1] & (1 << 29)) != 0; - features.avx512bw = (regs[1] & (1 << 30)) != 0; - features.avx512vl = (regs[1] & (1 << 31)) != 0; + features.avx512f = (regs[1] & (1 << 16)) != 0; + features.avx512dq = (regs[1] & (1 << 29)) != 0; + features.avx512bw = (regs[1] & (1 << 30)) != 0; + features.avx512vl = (regs[1] & (1 << 31)) != 0; features.avx512vbmi = (regs[2] & (1 << 1)) != 0; } #endif @@ -99,10 +97,10 @@ inline CpuFeatures detect_cpu_features() { #endif #if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) - const unsigned long hwcap = getauxval(AT_HWCAP); + const unsigned long hwcap = getauxval(AT_HWCAP); const unsigned long hwcap2 = getauxval(AT_HWCAP2); - features.sve = (hwcap & HWCAP_SVE) != 0; - features.sve2 = (hwcap2 & HWCAP2_SVE2) != 0; + features.sve = (hwcap & HWCAP_SVE) != 0; + features.sve2 = (hwcap2 & HWCAP2_SVE2) != 0; #endif return features; @@ -113,6 +111,6 @@ inline CpuFeatures get_cached_cpu_features() { return features; } #endif -} +} // namespace pernix::internal -#endif //PERNIX_CPU_FEATURES_H +#endif // PERNIX_CPU_FEATURES_H diff --git a/include/pernix/dispatch/kernel.h b/include/pernix/dispatch/kernel.h index 47198c3..aafdab1 100644 --- a/include/pernix/dispatch/kernel.h +++ b/include/pernix/dispatch/kernel.h @@ -2,6 +2,7 @@ #define PERNIX_KERNEL_H #include + #include namespace pernix::internal { @@ -15,13 +16,10 @@ struct Kernel { std::string_view name; FuncType func; - explicit operator bool() const noexcept { - return func != nullptr; - } + explicit operator bool() const noexcept { return func != nullptr; } - Kernel(const std::string_view kernel_name, FuncType kernel_func) : name(kernel_name), func(kernel_func) { - } + Kernel(const std::string_view kernel_name, FuncType kernel_func) : name(kernel_name), func(kernel_func) {} }; -} +} // namespace pernix::internal -#endif //PERNIX_KERNEL_H +#endif // PERNIX_KERNEL_H diff --git a/include/pernix/dispatch/select.h b/include/pernix/dispatch/select.h index 33ed123..8e723a2 100644 --- a/include/pernix/dispatch/select.h +++ b/include/pernix/dispatch/select.h @@ -1,9 +1,10 @@ #ifndef PERNIX_SELECT_H #define PERNIX_SELECT_H -#include #include +#include + #if !defined(PERNIX_BUILD_X86_AVX2) && !defined(PERNIX_DISABLE_AVX2) #if defined(PERNIX_USE_SIMDE) || defined(__AVX2__) #define PERNIX_BUILD_X86_AVX2 1 @@ -17,7 +18,8 @@ #endif #if !defined(PERNIX_BUILD_X86_AVX512_VBMI) && !defined(PERNIX_DISABLE_AVX512) -#if defined(PERNIX_USE_SIMDE) || (defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && defined(__AVX512VL__) && defined(__AVX512VBMI__)) +#if defined(PERNIX_USE_SIMDE) || \ + (defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && defined(__AVX512VL__) && defined(__AVX512VBMI__)) #define PERNIX_BUILD_X86_AVX512_VBMI 1 #endif #endif @@ -35,231 +37,205 @@ #endif namespace pernix::internal { - Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size); - - Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size); - - Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size); +Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size); - Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size); +Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size); - Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size); - Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size); - Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values); - Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values); +Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values); - Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values); - Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_fallback_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_fallback_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_fallback_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_fallback_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_fallback_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_fallback_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_fallback_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel - select_fallback_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_fallback_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_decompress_blocks_f32(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_fallback_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel - select_fallback_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_fallback_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_decompress_blocks_f64(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_fallback_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_scalar_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_fallback_scalar_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_fallback_scalar_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_fallback_scalar_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_fallback_scalar_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_fallback_scalar_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_fallback_scalar_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_fallback_scalar_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel - select_fallback_scalar_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_fallback_scalar_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_scalar_decompress_blocks_f32(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_fallback_scalar_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel - select_fallback_scalar_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_fallback_scalar_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_scalar_decompress_blocks_f64(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_fallback_scalar_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_stdpar_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_fallback_stdpar_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_fallback_stdpar_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_fallback_stdpar_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_fallback_stdpar_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_fallback_stdpar_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_fallback_stdpar_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_fallback_stdpar_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel - select_fallback_stdpar_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_fallback_stdpar_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_stdpar_decompress_blocks_f32(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_fallback_stdpar_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel - select_fallback_stdpar_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_fallback_stdpar_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_stdpar_decompress_blocks_f64(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_fallback_stdpar_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_simd_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_fallback_simd_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_fallback_simd_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_fallback_simd_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_fallback_simd_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_fallback_simd_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_fallback_simd_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_fallback_simd_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel - select_fallback_simd_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_fallback_simd_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_simd_decompress_blocks_f32(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_fallback_simd_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel - select_fallback_simd_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_fallback_simd_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_fallback_simd_decompress_blocks_f64(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_fallback_simd_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #if defined(PERNIX_BUILD_X86_AVX2) - Kernel select_avx2_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_avx2_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_avx2_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_avx2_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_avx2_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_avx2_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_avx2_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_avx2_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel select_avx2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_avx2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_avx2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_avx2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_avx2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_avx2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_avx2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_avx2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - Kernel select_bmi2_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_bmi2_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_bmi2_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_bmi2_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_bmi2_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_bmi2_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_bmi2_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_bmi2_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel select_bmi2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_bmi2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_bmi2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_bmi2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_bmi2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_bmi2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_bmi2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_bmi2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - Kernel select_avx512vbmi_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_avx512vbmi_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_avx512vbmi_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_avx512vbmi_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_avx512vbmi_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_avx512vbmi_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_avx512vbmi_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_avx512vbmi_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel select_avx512vbmi_decompress_block_f32(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_avx512vbmi_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_avx512vbmi_decompress_blocks_f32(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_avx512vbmi_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_avx512vbmi_decompress_block_f64(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_avx512vbmi_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_avx512vbmi_decompress_blocks_f64(u8 bit_width, u32 block_size, - bool sign_values); +Kernel select_avx512vbmi_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - Kernel select_neon_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_neon_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_neon_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_neon_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_neon_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_neon_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_neon_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_neon_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel select_neon_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_neon_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_neon_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_neon_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_neon_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_neon_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_neon_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_neon_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - Kernel select_sve2_compress_block_f32(u8 bit_width, u32 block_size); +Kernel select_sve2_compress_block_f32(u8 bit_width, u32 block_size); - Kernel select_sve2_compress_blocks_f32(u8 bit_width, u32 block_size); +Kernel select_sve2_compress_blocks_f32(u8 bit_width, u32 block_size); - Kernel select_sve2_compress_block_f64(u8 bit_width, u32 block_size); +Kernel select_sve2_compress_block_f64(u8 bit_width, u32 block_size); - Kernel select_sve2_compress_blocks_f64(u8 bit_width, u32 block_size); +Kernel select_sve2_compress_blocks_f64(u8 bit_width, u32 block_size); - Kernel select_sve2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_sve2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_sve2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_sve2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_sve2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_sve2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); - Kernel select_sve2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); +Kernel select_sve2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); #endif -} +} // namespace pernix::internal #if !defined(PERNIX_BUILD_LIB) #include #endif -#endif //PERNIX_SELECT_H +#endif // PERNIX_SELECT_H diff --git a/include/pernix/fallback/simd_compression.h b/include/pernix/fallback/simd_compression.h index 0658cbd..d87e8f6 100644 --- a/include/pernix/fallback/simd_compression.h +++ b/include/pernix/fallback/simd_compression.h @@ -16,9 +16,9 @@ namespace pernix { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int compress_block_fallback_simd(const void *input_ptr, ScaleType scale, void *output_ptr) { +int compress_block_fallback_simd(const void* input_ptr, ScaleType scale, void* output_ptr) { #if defined(__cpp_lib_simd) - using simd_placeholder = std::simd; + using simd_placeholder = std::simd; [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); #endif [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; @@ -31,9 +31,9 @@ int compress_block_fallback_simd(const void *input_ptr, ScaleType scale, void *o template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int compress_blocks_fallback_simd(const void *input_ptr, ScaleType scale, void *output_ptr, u32 blocks) { +int compress_blocks_fallback_simd(const void* input_ptr, ScaleType scale, void* output_ptr, u32 blocks) { #if defined(__cpp_lib_simd) - using simd_placeholder = std::simd; + using simd_placeholder = std::simd; [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); #endif static_cast(input_ptr); @@ -42,6 +42,6 @@ int compress_blocks_fallback_simd(const void *input_ptr, ScaleType scale, void * static_cast(blocks); return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; } -} +} // namespace pernix -#endif // PERNIX_FALLBACK_SIMD_COMPRESSION_H +#endif // PERNIX_FALLBACK_SIMD_COMPRESSION_H diff --git a/include/pernix/fallback/simd_decompression.h b/include/pernix/fallback/simd_decompression.h index 4ec8f1c..a876948 100644 --- a/include/pernix/fallback/simd_decompression.h +++ b/include/pernix/fallback/simd_decompression.h @@ -16,12 +16,12 @@ namespace pernix { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int decompress_block_fallback_simd(const void *input_ptr, ScaleType scale, void *output_ptr) { +int decompress_block_fallback_simd(const void* input_ptr, ScaleType scale, void* output_ptr) { #if defined(__cpp_lib_simd) - using simd_placeholder = std::simd; + using simd_placeholder = std::simd; [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); #endif - [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; + [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; static_cast(input_ptr); static_cast(scale); @@ -33,9 +33,9 @@ int decompress_block_fallback_simd(const void *input_ptr, ScaleType scale, void template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int decompress_blocks_fallback_simd(const void *input_ptr, ScaleType scale, void *output_ptr, u32 blocks) { +int decompress_blocks_fallback_simd(const void* input_ptr, ScaleType scale, void* output_ptr, u32 blocks) { #if defined(__cpp_lib_simd) - using simd_placeholder = std::simd; + using simd_placeholder = std::simd; [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); #endif [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; @@ -46,6 +46,6 @@ int decompress_blocks_fallback_simd(const void *input_ptr, ScaleType scale, void static_cast(sign_values); return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; } -} +} // namespace pernix -#endif // PERNIX_FALLBACK_SIMD_DECOMPRESSION_H +#endif // PERNIX_FALLBACK_SIMD_DECOMPRESSION_H diff --git a/include/pernix/fallback/stdpar_common.h b/include/pernix/fallback/stdpar_common.h index cc179d8..8d7a5bd 100644 --- a/include/pernix/fallback/stdpar_common.h +++ b/include/pernix/fallback/stdpar_common.h @@ -72,16 +72,11 @@ class counting_iterator { using pointer = void; using reference = T; - constexpr explicit counting_iterator(T value) noexcept : value_(value) { - } + constexpr explicit counting_iterator(T value) noexcept : value_(value) {} - constexpr T operator*() const noexcept { - return value_; - } + constexpr T operator*() const noexcept { return value_; } - constexpr T operator[](difference_type n) const noexcept { - return value_ + n; - } + constexpr T operator[](difference_type n) const noexcept { return value_ + n; } constexpr counting_iterator& operator++() noexcept { ++value_; @@ -134,33 +129,21 @@ class counting_iterator { return static_cast(lhs.value_) - static_cast(rhs.value_); } - friend constexpr bool operator==(counting_iterator lhs, counting_iterator rhs) noexcept { - return lhs.value_ == rhs.value_; - } + friend constexpr bool operator==(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ == rhs.value_; } - friend constexpr bool operator!=(counting_iterator lhs, counting_iterator rhs) noexcept { - return !(lhs == rhs); - } + friend constexpr bool operator!=(counting_iterator lhs, counting_iterator rhs) noexcept { return !(lhs == rhs); } - friend constexpr bool operator<(counting_iterator lhs, counting_iterator rhs) noexcept { - return lhs.value_ < rhs.value_; - } + friend constexpr bool operator<(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ < rhs.value_; } - friend constexpr bool operator<=(counting_iterator lhs, counting_iterator rhs) noexcept { - return lhs.value_ <= rhs.value_; - } + friend constexpr bool operator<=(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ <= rhs.value_; } - friend constexpr bool operator>(counting_iterator lhs, counting_iterator rhs) noexcept { - return lhs.value_ > rhs.value_; - } + friend constexpr bool operator>(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ > rhs.value_; } - friend constexpr bool operator>=(counting_iterator lhs, counting_iterator rhs) noexcept { - return lhs.value_ >= rhs.value_; - } + friend constexpr bool operator>=(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ >= rhs.value_; } private: T value_; }; -} +} // namespace pernix::internal -#endif // PERNIX_FALLBACK_STDPAR_COMMON_H +#endif // PERNIX_FALLBACK_STDPAR_COMMON_H diff --git a/include/pernix/fallback/stdpar_compression.h b/include/pernix/fallback/stdpar_compression.h index 5f494d9..f8aab9e 100644 --- a/include/pernix/fallback/stdpar_compression.h +++ b/include/pernix/fallback/stdpar_compression.h @@ -13,8 +13,8 @@ namespace pernix { namespace internal { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && GROUP_SIZE >= 1 && GROUP_SIZE <= 8 && std::is_floating_point_v) -__always_inline stdpar_buffer -quantize_epi32_group_fallback_stdpar(const std::span input, const ScaleType scale) { +__always_inline stdpar_buffer quantize_epi32_group_fallback_stdpar(const std::span input, + const ScaleType scale) { stdpar_buffer output; #pragma GCC unroll GROUP_SIZE @@ -28,8 +28,7 @@ quantize_epi32_group_fallback_stdpar(const std::span input, con template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && GROUP_SIZE >= 1 && GROUP_SIZE <= 8) -__always_inline void pack_epi32_group_fallback_stdpar(const stdpar_buffer& input, - const std::span destination) { +__always_inline void pack_epi32_group_fallback_stdpar(const stdpar_buffer& input, const std::span destination) { constexpr u32 bitmask = (u32{1} << BIT_WIDTH) - 1U; usize idx = 0; @@ -38,13 +37,13 @@ __always_inline void pack_epi32_group_fallback_stdpar(const stdpar_buffer(input.data[i] & bitmask) << bits_in_buffer); + buffer |= (static_cast(input.data[i] & bitmask) << bits_in_buffer); bits_in_buffer += BIT_WIDTH; while (bits_in_buffer >= 8) { destination[idx++] = static_cast(buffer & 0xFFU); - buffer >>= 8; - bits_in_buffer -= 8; + buffer >>= 8; + bits_in_buffer -= 8; } } @@ -52,7 +51,7 @@ __always_inline void pack_epi32_group_fallback_stdpar(const stdpar_buffer(buffer & 0xFFU); } } -} +} // namespace internal template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) @@ -85,8 +84,8 @@ int compress_block_fallback_stdpar(const void* input_ptr, ScaleType scale, void* const auto quantized = internal::quantize_epi32_group_fallback_stdpar( std::span(input + tail_input_offset, tail_group_size), scale); - internal::pack_epi32_group_fallback_stdpar( - quantized, std::span(output + tail_output_offset, tail_bytes)); + internal::pack_epi32_group_fallback_stdpar(quantized, + std::span(output + tail_output_offset, tail_bytes)); } return PERNIX_STATUS_OK; @@ -96,7 +95,7 @@ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) int compress_blocks_fallback_stdpar(const void* input_ptr, ScaleType scale, void* output_ptr, u32 blocks) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; - + const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -104,13 +103,11 @@ int compress_blocks_fallback_stdpar(const void* input_ptr, ScaleType scale, void std::for_each_n(std::execution::par_unseq, first, blocks, [&](const u32 block) { const auto* block_input = input + (static_cast(block) * elements_per_block); auto* block_output = output + (static_cast(block) * BLOCK_SIZE); - static_cast(compress_block_fallback_stdpar(block_input, - scale, - block_output)); + static_cast(compress_block_fallback_stdpar(block_input, scale, block_output)); }); return PERNIX_STATUS_OK; } -} +} // namespace pernix -#endif // PERNIX_FALLBACK_STDPAR_COMPRESSION_H +#endif // PERNIX_FALLBACK_STDPAR_COMPRESSION_H diff --git a/include/pernix/fallback/stdpar_decompression.h b/include/pernix/fallback/stdpar_decompression.h index ffbae89..da85797 100644 --- a/include/pernix/fallback/stdpar_decompression.h +++ b/include/pernix/fallback/stdpar_decompression.h @@ -15,7 +15,7 @@ namespace internal { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && GROUP_SIZE >= 1 && GROUP_SIZE <= 8) __always_inline stdpar_buffer unpack_epi32_group_fallback_stdpar(const std::span input) { - constexpr auto bit_offset = 0; // TODO: for now + constexpr auto bit_offset = 0; // TODO: for now constexpr u32 bitmask = (u32{1} << BIT_WIDTH) - 1U; stdpar_buffer output; @@ -30,8 +30,8 @@ __always_inline stdpar_buffer unpack_epi32_group_fallback_stdpa for (usize i = 0; i < GROUP_SIZE; i++) { while (BIT_WIDTH > bits_in_buffer) { const auto next_value = static_cast(input[idx++]) << bits_in_buffer; - buffer |= next_value; - bits_in_buffer += 8; + buffer |= next_value; + bits_in_buffer += 8; } const u32 raw_value = static_cast(buffer & bitmask); @@ -42,17 +42,16 @@ __always_inline stdpar_buffer unpack_epi32_group_fallback_stdpa } ++output.size; - buffer >>= BIT_WIDTH; + buffer >>= BIT_WIDTH; bits_in_buffer -= BIT_WIDTH; } return output; } -} +} // namespace internal template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int decompress_block_fallback_stdpar(const void* __restrict__ input_ptr, const ScaleType scale, - void* __restrict__ output_ptr) { +int decompress_block_fallback_stdpar(const void* __restrict__ input_ptr, const ScaleType scale, void* __restrict__ output_ptr) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 full_groups = elements_per_block / 8; constexpr u32 remainder = elements_per_block % 8; @@ -73,17 +72,13 @@ int decompress_block_fallback_stdpar(const void* __restrict__ input_ptr, const S if constexpr (remainder != 0) { constexpr usize tail_input_offset = full_groups * BIT_WIDTH; constexpr usize tail_output_offset = full_groups * 8U; - constexpr usize tail_bytes = internal::packed_group_bytes_v(remainder), BIT_WIDTH>; + constexpr usize tail_bytes = internal::packed_group_bytes_v(remainder), BIT_WIDTH>; - const auto unpacked = internal::unpack_epi32_group_fallback_stdpar< - BIT_WIDTH, - SIGN_VALUES, - static_cast(remainder)>( + const auto unpacked = internal::unpack_epi32_group_fallback_stdpar(remainder)>( input_span.subspan(tail_input_offset, tail_bytes)); for (u32 i = 0; i < remainder; ++i) { - output_span[tail_output_offset + i] = - internal::dequantize_stdpar_value(unpacked.data[i], scale); + output_span[tail_output_offset + i] = internal::dequantize_stdpar_value(unpacked.data[i], scale); } } @@ -92,8 +87,8 @@ int decompress_block_fallback_stdpar(const void* __restrict__ input_ptr, const S template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int decompress_blocks_fallback_stdpar(const void* __restrict__ input_ptr, const ScaleType scale, - void* __restrict__ output_ptr, const u32 blocks) { +int decompress_blocks_fallback_stdpar(const void* __restrict__ input_ptr, const ScaleType scale, void* __restrict__ output_ptr, + const u32 blocks) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const auto* input = static_cast(input_ptr); @@ -103,12 +98,12 @@ int decompress_blocks_fallback_stdpar(const void* __restrict__ input_ptr, const std::for_each_n(std::execution::par_unseq, first, blocks, [&](const u32 block) { const auto* block_input = input + (static_cast(block) * BLOCK_SIZE); auto* block_output = output + (static_cast(block) * elements_per_block); - static_cast(decompress_block_fallback_stdpar( - block_input, scale, block_output)); + static_cast( + decompress_block_fallback_stdpar(block_input, scale, block_output)); }); return 0; } -} +} // namespace pernix -#endif // PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H +#endif // PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H diff --git a/include/pernix/pernix.h b/include/pernix/pernix.h index 51b9c75..2609f7d 100644 --- a/include/pernix/pernix.h +++ b/include/pernix/pernix.h @@ -12,25 +12,25 @@ extern "C" { #endif typedef enum pernix_status { - PERNIX_STATUS_OK = 0, - PERNIX_STATUS_INVALID_ARGUMENT = -1, - PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH = -2, - PERNIX_STATUS_UNSUPPORTED_BACKEND = -3, - PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE = -4, + PERNIX_STATUS_OK = 0, + PERNIX_STATUS_INVALID_ARGUMENT = -1, + PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH = -2, + PERNIX_STATUS_UNSUPPORTED_BACKEND = -3, + PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE = -4, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION = -5, } pernix_status; typedef enum pernix_backend { - PERNIX_BACKEND_AUTO = 0, - PERNIX_BACKEND_FALLBACK = 1, + PERNIX_BACKEND_AUTO = 0, + PERNIX_BACKEND_FALLBACK = 1, PERNIX_BACKEND_FALLBACK_SCALAR = PERNIX_BACKEND_FALLBACK, - PERNIX_BACKEND_X86_AVX2 = 2, - PERNIX_BACKEND_X86_BMI2 = 3, + PERNIX_BACKEND_X86_AVX2 = 2, + PERNIX_BACKEND_X86_BMI2 = 3, PERNIX_BACKEND_X86_AVX512_VBMI = 4, - PERNIX_BACKEND_ARM64_NEON = 5, - PERNIX_BACKEND_ARM64_SVE = 6, + PERNIX_BACKEND_ARM64_NEON = 5, + PERNIX_BACKEND_ARM64_SVE = 6, PERNIX_BACKEND_FALLBACK_STDPAR = 7, - PERNIX_BACKEND_FALLBACK_SIMD = 8 + PERNIX_BACKEND_FALLBACK_SIMD = 8 } pernix_backend; PERNIX_API u8 pernix_min_bit_width(void); @@ -71,40 +71,32 @@ PERNIX_API pernix_status pernix_compression_scale_f32(float bmax, u8 bit_width, /* Computes the inverse compression scale directly from bmax and bit width. */ PERNIX_API pernix_status pernix_compression_scale_f64(double bmax, u8 bit_width, double* inverse_scale); -PERNIX_API pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, - const void* input, +PERNIX_API pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float inverse_scale, void* output); -PERNIX_API pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, - const void* input, +PERNIX_API pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float inverse_scale, void* output, u32 blocks); -PERNIX_API pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, - const void* input, - float scale, void* output, bool sign_values); +PERNIX_API pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output, bool sign_values); -PERNIX_API pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, - const void* input, - float scale, void* output, u32 blocks, bool sign_values); +PERNIX_API pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output, u32 blocks, bool sign_values); -PERNIX_API pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, - const void* input, +PERNIX_API pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double inverse_scale, void* output); -PERNIX_API pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, - const void* input, +PERNIX_API pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double inverse_scale, void* output, u32 blocks); -PERNIX_API pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, - const void* input, - double scale, void* output, bool sign_values); +PERNIX_API pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, bool sign_values); -PERNIX_API pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, - const void* input, - double scale, void* output, u32 blocks, bool sign_values); +PERNIX_API pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, u32 blocks, bool sign_values); #if defined(__cplusplus) } #endif -#endif //PERNIX_H +#endif // PERNIX_H diff --git a/include/pernix/pernix.hpp b/include/pernix/pernix.hpp index c4f3c46..78fc1de 100644 --- a/include/pernix/pernix.hpp +++ b/include/pernix/pernix.hpp @@ -1,10 +1,9 @@ #ifndef PERNIX_HPP #define PERNIX_HPP +#include #include #include - -#include #include namespace pernix { @@ -35,35 +34,33 @@ constexpr u32 elements_per_block(const u8 bit_width) { } template -__always_inline Status scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT &scale) { +__always_inline Status scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT& scale) { return detail::scale_from_bmax(bmax, bit_width, &scale); } template -__always_inline Status decompression_scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT &scale) { +__always_inline Status decompression_scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT& scale) { return scale_from_bmax(bmax, bit_width, scale); } template -__always_inline Status inverse_scale(const FloatT scale, FloatT &inverse_scale_value) { +__always_inline Status inverse_scale(const FloatT scale, FloatT& inverse_scale_value) { return detail::inverse_scale(scale, &inverse_scale_value); } template -__always_inline Status compression_scale_from_bmax(const FloatT bmax, const u8 bit_width, - FloatT &inverse_scale_value) { +__always_inline Status compression_scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT& inverse_scale_value) { return detail::compression_scale_from_bmax(bmax, bit_width, &inverse_scale_value); } -__always_inline const char *status_string(const Status status) { +__always_inline const char* status_string(const Status status) { return detail::status_string(status); } namespace detail { template -__always_inline Status validate_compress_spans(const u8 bit_width, const u32 block_size, - const std::span input, const std::span output, - const u32 blocks) { +__always_inline Status validate_compress_spans(const u8 bit_width, const u32 block_size, const std::span input, + const std::span output, const u32 blocks) { if (!is_valid_bit_width(bit_width)) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; } @@ -74,16 +71,14 @@ __always_inline Status validate_compress_spans(const u8 bit_width, const u32 blo return PERNIX_STATUS_INVALID_ARGUMENT; } - const usize required_input = static_cast(elements_per_block(bit_width, block_size)) * blocks; + const usize required_input = static_cast(elements_per_block(bit_width, block_size)) * blocks; const usize required_output = static_cast(block_size) * blocks; - return input.size() >= required_input && output.size() >= required_output ? PERNIX_STATUS_OK - : PERNIX_STATUS_INVALID_ARGUMENT; + return input.size() >= required_input && output.size() >= required_output ? PERNIX_STATUS_OK : PERNIX_STATUS_INVALID_ARGUMENT; } template -__always_inline Status validate_decompress_spans(const u8 bit_width, const u32 block_size, - const std::span input, const std::span output, - const u32 blocks) { +__always_inline Status validate_decompress_spans(const u8 bit_width, const u32 block_size, const std::span input, + const std::span output, const u32 blocks) { if (!is_valid_bit_width(bit_width)) { return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; } @@ -94,93 +89,74 @@ __always_inline Status validate_decompress_spans(const u8 bit_width, const u32 b return PERNIX_STATUS_INVALID_ARGUMENT; } - const usize required_input = static_cast(block_size) * blocks; + const usize required_input = static_cast(block_size) * blocks; const usize required_output = static_cast(elements_per_block(bit_width, block_size)) * blocks; - return input.size() >= required_input && output.size() >= required_output ? PERNIX_STATUS_OK - : PERNIX_STATUS_INVALID_ARGUMENT; + return input.size() >= required_input && output.size() >= required_output ? PERNIX_STATUS_OK : PERNIX_STATUS_INVALID_ARGUMENT; } -} // namespace detail +} // namespace detail -__always_inline Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const float inverse_scale, - std::span output) { - if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); - status != PERNIX_STATUS_OK) { +__always_inline Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, std::span output) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::compress_block(backend, bit_width, block_size, input.data(), inverse_scale, output.data()); } -__always_inline Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const double inverse_scale, - std::span output) { - if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); - status != PERNIX_STATUS_OK) { +__always_inline Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, std::span output) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::compress_block(backend, bit_width, block_size, input.data(), inverse_scale, output.data()); } -__always_inline Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const float scale, std::span output, - const bool sign_values = true) { - if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); - status != PERNIX_STATUS_OK) { +__always_inline Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float scale, std::span output, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } -__always_inline Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const double scale, std::span output, - const bool sign_values = true) { - if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); - status != PERNIX_STATUS_OK) { +__always_inline Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double scale, std::span output, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } -__always_inline Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const float inverse_scale, - std::span output, const u32 blocks) { - if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); - status != PERNIX_STATUS_OK) { +__always_inline Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, std::span output, const u32 blocks) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } return detail::compress_blocks(backend, bit_width, block_size, input.data(), inverse_scale, output.data(), blocks); } -__always_inline Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const double inverse_scale, - std::span output, const u32 blocks) { - if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); - status != PERNIX_STATUS_OK) { +__always_inline Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, std::span output, const u32 blocks) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } return detail::compress_blocks(backend, bit_width, block_size, input.data(), inverse_scale, output.data(), blocks); } -__always_inline Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const float scale, std::span output, - const u32 blocks, const bool sign_values = true) { - if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); - status != PERNIX_STATUS_OK) { +__always_inline Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float scale, std::span output, const u32 blocks, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } - return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, - sign_values); + return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, sign_values); } -__always_inline Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, - const std::span input, const double scale, std::span output, - const u32 blocks, const bool sign_values = true) { - if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); - status != PERNIX_STATUS_OK) { +__always_inline Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double scale, std::span output, const u32 blocks, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } - return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, - sign_values); + return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, sign_values); } // convenience overloads without backend (defaults to Auto) @@ -194,15 +170,13 @@ __always_inline Status compress_block(const u8 bit_width, const u32 block_size, return compress_block(Backend::Auto, bit_width, block_size, input, inverse_scale, output); } -__always_inline Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, - const float scale, const std::span output, - const bool sign_values = true) { +__always_inline Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, const float scale, + const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); } -__always_inline Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, - const double scale, const std::span output, - const bool sign_values = true) { +__always_inline Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, const double scale, + const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); } @@ -216,26 +190,24 @@ __always_inline Status compress_blocks(const u8 bit_width, const u32 block_size, return compress_blocks(Backend::Auto, bit_width, block_size, input, inverse_scale, output, blocks); } -__always_inline Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, - const float scale, const std::span output, const u32 blocks, - const bool sign_values = true) { +__always_inline Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, const float scale, + const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); } -__always_inline Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, - const double scale, const std::span output, const u32 blocks, - const bool sign_values = true) { +__always_inline Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, const double scale, + const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); } // convenience overloads without backend and without block_size (defaults to 64) -__always_inline Status compress_block(const u8 bit_width, const std::span input, - const float inverse_scale, const std::span output) { +__always_inline Status compress_block(const u8 bit_width, const std::span input, const float inverse_scale, + const std::span output) { return compress_block(Backend::Auto, bit_width, 64, input, inverse_scale, output); } -__always_inline Status compress_block(const u8 bit_width, const std::span input, - const double inverse_scale, const std::span output) { +__always_inline Status compress_block(const u8 bit_width, const std::span input, const double inverse_scale, + const std::span output) { return compress_block(Backend::Auto, bit_width, 64, input, inverse_scale, output); } @@ -249,27 +221,25 @@ __always_inline Status decompress_block(const u8 bit_width, const std::span input, - const float inverse_scale, const std::span output, const u32 blocks) { +__always_inline Status compress_blocks(const u8 bit_width, const std::span input, const float inverse_scale, + const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, 64, input, inverse_scale, output, blocks); } -__always_inline Status compress_blocks(const u8 bit_width, const std::span input, - const double inverse_scale, const std::span output, const u32 blocks) { +__always_inline Status compress_blocks(const u8 bit_width, const std::span input, const double inverse_scale, + const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, 64, input, inverse_scale, output, blocks); } __always_inline Status decompress_blocks(const u8 bit_width, const std::span input, const float scale, - const std::span output, const u32 blocks, - const bool sign_values = true) { + const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); } __always_inline Status decompress_blocks(const u8 bit_width, const std::span input, const double scale, - const std::span output, const u32 blocks, - const bool sign_values = true) { + const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); } -} // namespace pernix +} // namespace pernix -#endif //PERNIX_HPP +#endif // PERNIX_HPP diff --git a/include/pernix/simd_compat.h b/include/pernix/simd_compat.h index d55aafa..dbc0154 100644 --- a/include/pernix/simd_compat.h +++ b/include/pernix/simd_compat.h @@ -2,6 +2,7 @@ #define PERNIX_SIMD_COMPAT_H #include + #include #include diff --git a/include/pernix/x86/avx2/avx2_compression.h b/include/pernix/x86/avx2/avx2_compression.h index e7e3e9a..ce8fd98 100644 --- a/include/pernix/x86/avx2/avx2_compression.h +++ b/include/pernix/x86/avx2/avx2_compression.h @@ -1,9 +1,9 @@ #ifndef PERNIX_AVX2_COMPRESSION_H #define PERNIX_AVX2_COMPRESSION_H -#include #include #include +#include #include #include @@ -18,18 +18,16 @@ template __always_inline __m256i mm256_clamp_signed_epi32(__m256i input) { constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), - _mm256_set1_epi32(max_value)); + return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), _mm256_set1_epi32(max_value)); } - /** -* @brief Quantize four float values into signed 32-bit integers. -* -* @param input source float lane values. -* @param scale per-lane scale factor. -* @return __m128i quantized values. -*/ + * @brief Quantize four float values into signed 32-bit integers. + * + * @param input source float lane values. + * @param scale per-lane scale factor. + * @return __m128i quantized values. + */ __always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& scale) { const __m128 scaled = _mm_mul_ps(input, scale); // const __m128 rounded = _mm_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); @@ -37,24 +35,24 @@ __always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& } /** -* @brief Quantize two double values into a partially filled 128-bit integer register. -* -* @param input source double lane values. -* @param scale per-lane scale factor. -* @return __m128i quantized values in the low lanes. -*/ + * @brief Quantize two double values into a partially filled 128-bit integer register. + * + * @param input source double lane values. + * @param scale per-lane scale factor. + * @return __m128i quantized values in the low lanes. + */ __always_inline __m128i mm_quantize_pd_epi32(const __m128d& input, const __m128d& scale) { const __m128d scaled = _mm_mul_pd(input, scale); return _mm_cvtpd_epi32(scaled); } /** -* @brief Quantize eight float values into signed 32-bit integers. -* -* @param input source float lane values. -* @param scale per-lane scale factor. -* @return __m256i quantized values. -*/ + * @brief Quantize eight float values into signed 32-bit integers. + * + * @param input source float lane values. + * @param scale per-lane scale factor. + * @return __m256i quantized values. + */ __always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m256& scale) { const __m256 scaled = _mm256_mul_ps(input, scale); // const __m256 rounded = _mm256_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); @@ -62,12 +60,12 @@ __always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m25 } /** -* @brief Quantize four double values into signed 32-bit integers. -* -* @param input source double lane values. -* @param scale per-lane scale factor. -* @return __m128i quantized values in the low lanes. -*/ + * @brief Quantize four double values into signed 32-bit integers. + * + * @param input source double lane values. + * @param scale per-lane scale factor. + * @return __m128i quantized values in the low lanes. + */ __always_inline __m128i mm256_quantize_pd_epi32(const __m256d& input, const __m256d& scale) { const __m256d scaled = _mm256_mul_pd(input, scale); return _mm256_cvtpd_epi32(scaled); @@ -75,12 +73,12 @@ __always_inline __m128i mm256_quantize_pd_epi32(const __m256d& input, const __m2 #ifndef PERNIX_USE_SIMDE /** -* @brief Emulate per-16-bit left shifts on AVX2. -* -* @param a source values. -* @param count per-lane shift amounts. -* @return __m128i shifted values. -*/ + * @brief Emulate per-16-bit left shifts on AVX2. + * + * @param a source values. + * @param count per-lane shift amounts. + * @return __m128i shifted values. + */ __always_inline static __m128i _mm_sllv_epi16(const __m128i a, const __m128i count) { const __m128i mask = _mm_set1_epi32(0xffff0000); const __m128i low_half = _mm_sllv_epi32(a, _mm_andnot_si128(mask, count)); @@ -89,8 +87,8 @@ __always_inline static __m128i _mm_sllv_epi16(const __m128i a, const __m128i cou } /** -* @brief Emulate per-16-bit right shifts on AVX2. -*/ + * @brief Emulate per-16-bit right shifts on AVX2. + */ __always_inline static __m128i _mm_srlv_epi16(const __m128i a, const __m128i count) { const __m128i mask = _mm_set1_epi32(0x0000ffff); const __m128i low_half = _mm_srlv_epi32(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); @@ -99,8 +97,8 @@ __always_inline static __m128i _mm_srlv_epi16(const __m128i a, const __m128i cou } /** -* @brief Emulate per-16-bit left shifts on 256-bit AVX2 vectors. -*/ + * @brief Emulate per-16-bit left shifts on 256-bit AVX2 vectors. + */ __always_inline static __m256i _mm256_sllv_epi16(const __m256i a, const __m256i count) { const __m256i mask = _mm256_set1_epi32(0xffff0000); const __m256i low_half = _mm256_sllv_epi32(a, _mm256_andnot_si256(mask, count)); @@ -109,8 +107,8 @@ __always_inline static __m256i _mm256_sllv_epi16(const __m256i a, const __m256i } /** -* @brief Emulate per-16-bit right shifts on 256-bit AVX2 vectors. -*/ + * @brief Emulate per-16-bit right shifts on 256-bit AVX2 vectors. + */ __always_inline static __m256i _mm256_srlv_epi16(const __m256i a, const __m256i count) { const __m256i mask = _mm256_set1_epi32(0x0000ffff); const __m256i low_half = _mm256_srlv_epi32(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); @@ -119,22 +117,22 @@ __always_inline static __m256i _mm256_srlv_epi16(const __m256i a, const __m256i } /** -* @brief Blend 8-bit lanes by expanding a scalar mask value. -*/ + * @brief Blend 8-bit lanes by expanding a scalar mask value. + */ __always_inline static __m128i mm_blend_epi8(const __m128i X, const __m128i Y, const i8 M) { return _mm_blendv_epi8(X, Y, _mm_set1_epi8(M)); } /** -* @brief Blend 8-bit lanes in 256-bit vectors by expanding a scalar mask value. -*/ + * @brief Blend 8-bit lanes in 256-bit vectors by expanding a scalar mask value. + */ __always_inline static __m256i mm256_blend_epi8(const __m256i X, const __m256i Y, const i8 M) { return _mm256_blendv_epi8(X, Y, _mm256_set1_epi8(M)); } /** -* @brief Emulate per-byte left shifts on 128-bit vectors. -*/ + * @brief Emulate per-byte left shifts on 128-bit vectors. + */ __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { const __m128i mask = _mm_set1_epi16(static_cast(0xff00u)); const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); @@ -143,8 +141,8 @@ __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i coun } /** -* @brief Emulate per-byte right shifts on 128-bit vectors. -*/ + * @brief Emulate per-byte right shifts on 128-bit vectors. + */ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i count) { const __m128i mask = _mm_set1_epi16(0x00ff); const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); @@ -153,8 +151,8 @@ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i coun } /** -* @brief Emulate per-byte left shifts on 256-bit vectors. -*/ + * @brief Emulate per-byte left shifts on 256-bit vectors. + */ __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { const __m256i mask = _mm256_set1_epi16(static_cast(0xff00u)); const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); @@ -163,8 +161,8 @@ __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i c } /** -* @brief Emulate per-byte right shifts on 256-bit vectors. -*/ + * @brief Emulate per-byte right shifts on 256-bit vectors. + */ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i count) { const __m256i mask = _mm256_set1_epi16(0x00ff); const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); @@ -174,8 +172,8 @@ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i c #endif /** -* @brief Pack four 32-bit values for bit widths 1 through 3. -*/ + * @brief Pack four 32-bit values for bit widths 1 through 3. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) __always_inline auto mm_pack_epi32_avx2_1to3(__m128i& input) -> __m128i { @@ -185,16 +183,15 @@ __always_inline auto mm_pack_epi32_avx2_1to3(__m128i& input) -> __m128i { alignas(16) u32 lanes[4]; _mm_storeu_si128(reinterpret_cast<__m128i*>(lanes), masked); - const u32 packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ( - (lanes[2] & bitmask) << (2 * BIT_WIDTH)) | + const u32 packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ((lanes[2] & bitmask) << (2 * BIT_WIDTH)) | ((lanes[3] & bitmask) << (3 * BIT_WIDTH)); return _mm_cvtsi32_si128(static_cast(packed)); } /** -* @brief Pack eight 32-bit values for bit widths 1 through 3. -*/ + * @brief Pack eight 32-bit values for bit widths 1 through 3. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) __always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { @@ -202,8 +199,7 @@ __always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(static_cast(bitmask))); - const __m256i shifts = _mm256_setr_epi32(0 * BIT_WIDTH, 1 * BIT_WIDTH, 2 * BIT_WIDTH, 3 * BIT_WIDTH, - 4 * BIT_WIDTH, 5 * BIT_WIDTH, + const __m256i shifts = _mm256_setr_epi32(0 * BIT_WIDTH, 1 * BIT_WIDTH, 2 * BIT_WIDTH, 3 * BIT_WIDTH, 4 * BIT_WIDTH, 5 * BIT_WIDTH, 6 * BIT_WIDTH, 7 * BIT_WIDTH); const __m256i shifted = _mm256_sllv_epi32(masked, shifts); @@ -225,17 +221,15 @@ __always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i& input) { const __m256i combined = _mm256_or_si256(packed8, _mm256_srli_epi16(packed8, 4)); - const __m256i shuffled = _mm256_shuffle_epi8(combined, _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 2, - 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1)); + const __m256i shuffled = _mm256_shuffle_epi8(combined, _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, + 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1)); return shuffled; } /** -* @brief Pack four 32-bit values for bit widths 9 through 16. -*/ + * @brief Pack four 32-bit values for bit widths 9 through 16. + */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { @@ -267,8 +261,8 @@ __always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { } /** -* @brief Pack eight 32-bit values for bit widths 9 through 16. -*/ + * @brief Pack eight 32-bit values for bit widths 9 through 16. + */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i& input) -> __m256i { @@ -319,8 +313,8 @@ __always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i } /** -* @brief Pack eight 32-bit values for bit widths 17 through 24. -*/ + * @brief Pack eight 32-bit values for bit widths 17 through 24. + */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m256i { @@ -353,8 +347,8 @@ __always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m25 } /** -* @brief Pack aligned 8-bit or 16-bit values from four 32-bit lanes. -*/ + * @brief Pack aligned 8-bit or 16-bit values from four 32-bit lanes. + */ template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { @@ -366,8 +360,8 @@ auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { } /** -* @brief Dispatch to the appropriate 128-bit AVX2 packer for the selected bit width. -*/ + * @brief Dispatch to the appropriate 128-bit AVX2 packer for the selected bit width. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 16) auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { @@ -384,25 +378,23 @@ auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { } /** -* @brief Pack aligned 8-bit or 16-bit values from eight 32-bit lanes. -*/ + * @brief Pack aligned 8-bit or 16-bit values from eight 32-bit lanes. + */ template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) __m256i mm256_pack_aligned_epi32_avx2(const __m256i& input) { if constexpr (BIT_WIDTH == 8) { - const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), - _mm256_extracti128_si256(input, 1)); - const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); + const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); + const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); return _mm256_castsi128_si256(packed8); } else { - return _mm256_castsi128_si256( - _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1))); + return _mm256_castsi128_si256(_mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1))); } } /** -* @brief Dispatch to the appropriate 256-bit AVX2 packer for the selected bit width. -*/ + * @brief Dispatch to the appropriate 256-bit AVX2 packer for the selected bit width. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __m256i mm256_pack_epi32_avx2(const __m256i& input) { @@ -436,25 +428,24 @@ void store_packed_epi32_fallback_8(const __m256i& input, u8* __restrict__ output } pack_epi32_fallback(block_values, output); } -} // namespace internal +} // namespace internal /** -* @brief Compress a single block of float using AVX2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). -* -* @param input pointer to the start of the input float values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX2 support. -*/ + * @brief Compress a single block of float using AVX2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). + * + * @param input pointer to the start of the input float values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr) { +int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -472,7 +463,7 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scal const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); internal::store_packed_epi32_fallback_8(packed_input, output); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -480,9 +471,7 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scal std::vector block_values(remaining); #pragma GCC unroll 8 for (u32 i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized( - internal::quantize_ps_epi32(input[i], scale))); + block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], scale))); } internal::pack_epi32_fallback(block_values, output); @@ -492,22 +481,21 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scal } /** -* @brief Compress a single block of double using AVX2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). -* -* @param input pointer to the start of the input double values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX2 support. -*/ + * @brief Compress a single block of double using AVX2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). + * + * @param input pointer to the start of the input double values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr) { +int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -520,15 +508,15 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scal const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 8 for (u32 iter = 0; iter < iterations_8; iter++) { - const __m256d source1 = _mm256_loadu_pd(input); - const __m256d source2 = _mm256_loadu_pd(input + 4); - const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); - const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); - __m256i combined = _mm256_castsi128_si256(quantized1); - combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256d source1 = _mm256_loadu_pd(input); + const __m256d source2 = _mm256_loadu_pd(input + 4); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); const __m256i packed_input = internal::mm256_clamp_signed_epi32(combined); internal::store_packed_epi32_fallback_8(packed_input, output); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -536,9 +524,7 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scal std::vector block_values(remaining); #pragma GCC unroll 8 for (u32 i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized( - internal::quantize_pd_epi64(input[i], scale))); + block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], scale))); } internal::pack_epi32_fallback(block_values, output); @@ -548,24 +534,22 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scal } /** -* @brief Compress multiple blocks using AVX2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). -* -* @param input pointer to the start of the input float values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @param blocks number of blocks to compress. -* @return int status code (0 for success). -* -* @note This function requires AVX2 support. -*/ + * @brief Compress multiple blocks using AVX2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). + * + * @param input pointer to the start of the input float values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @param blocks number of blocks to compress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr, - const u32 blocks) { +int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -574,7 +558,7 @@ int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f32 sca for (u32 block = 0; block < blocks; block++) { mm256_compress_block_avx2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } @@ -582,24 +566,22 @@ int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f32 sca } /** -* @brief Compress multiple blocks using AVX2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). -* -* @param input pointer to the start of the input double values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @param blocks number of blocks to compress. -* @return int status code (0 for success). -* -* @note This function requires AVX2 support. -*/ + * @brief Compress multiple blocks using AVX2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). + * + * @param input pointer to the start of the input double values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @param blocks number of blocks to compress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr, - const u32 blocks) { +int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -608,12 +590,12 @@ int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f64 sca for (u32 block = 0; block < blocks; block++) { mm256_compress_block_avx2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } return 0; } -} // namespace pernix +} // namespace pernix #endif // PERNIX_AVX2_COMPRESSION_H diff --git a/include/pernix/x86/avx2/avx2_decompression.h b/include/pernix/x86/avx2/avx2_decompression.h index f5af160..11e8978 100644 --- a/include/pernix/x86/avx2/avx2_decompression.h +++ b/include/pernix/x86/avx2/avx2_decompression.h @@ -1,9 +1,9 @@ #ifndef PERNIX_AVX2_DECOMPRESSION_H #define PERNIX_AVX2_DECOMPRESSION_H -#include #include #include +#include #include #include @@ -13,27 +13,23 @@ namespace pernix { namespace internal { /** -* @brief Convert an 8-lane mask to the lane representation used by AVX2 float masked stores. -*/ + * @brief Convert an 8-lane mask to the lane representation used by AVX2 float masked stores. + */ __always_inline __m256i mm256_convert_vmask_epi32(const __mmask8 mask8) { - return _mm256_setr_epi32((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, - (mask8 & 0x8) ? -1 : 0, - (mask8 & 0x10) ? -1 : 0, (mask8 & 0x20) ? -1 : 0, (mask8 & 0x40) ? -1 : 0, - (mask8 & 0x80) ? -1 : 0); + return _mm256_setr_epi32((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, (mask8 & 0x8) ? -1 : 0, + (mask8 & 0x10) ? -1 : 0, (mask8 & 0x20) ? -1 : 0, (mask8 & 0x40) ? -1 : 0, (mask8 & 0x80) ? -1 : 0); } /** -* @brief Convert a 4-lane mask to the lane representation used by AVX2 double masked stores. -*/ + * @brief Convert a 4-lane mask to the lane representation used by AVX2 double masked stores. + */ __always_inline __m256i mm256_convert_vmask_epi64(const __mmask8 mask8) { - return _mm256_setr_epi64x((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, - (mask8 & 0x8) ? -1 : 0); + return _mm256_setr_epi64x((mask8 & 0x1) ? -1 : 0, (mask8 & 0x2) ? -1 : 0, (mask8 & 0x4) ? -1 : 0, (mask8 & 0x8) ? -1 : 0); } - /** -* @brief Dequantize four 32-bit integers to floats. -*/ + * @brief Dequantize four 32-bit integers to floats. + */ __always_inline __m128 mm_dequantize_epi32(const __m128i& input, const __m128& scale) { const __m128 converted = _mm_cvtepi32_ps(input); return _mm_mul_ps(converted, scale); @@ -43,23 +39,23 @@ __always_inline __m128 mm_dequantize_epi32(const __m128i& input, const __m128& s __always_inline __m128d convert_epi64_pd(const __m128i v) { __m128i xH = _mm_srai_epi32(v, 16); xH = _mm_blend_epi16(xH, _mm_setzero_si128(), 0x33); - xH = _mm_add_epi64(xH, _mm_castpd_si128(_mm_set1_pd(442721857769029238784.))); // 3*2^67 - const __m128i xL = _mm_blend_epi16(v, _mm_castpd_si128(_mm_set1_pd(0x0010000000000000)), 0x88); // 2^52 - const __m128d f = _mm_sub_pd(_mm_castsi128_pd(xH), _mm_set1_pd(442726361368656609280.)); // 3*2^67 + 2^52 + xH = _mm_add_epi64(xH, _mm_castpd_si128(_mm_set1_pd(442721857769029238784.))); // 3*2^67 + const __m128i xL = _mm_blend_epi16(v, _mm_castpd_si128(_mm_set1_pd(0x0010000000000000)), 0x88); // 2^52 + const __m128d f = _mm_sub_pd(_mm_castsi128_pd(xH), _mm_set1_pd(442726361368656609280.)); // 3*2^67 + 2^52 return _mm_add_pd(f, _mm_castsi128_pd(xL)); } /** -* @brief Dequantize two 64-bit integers to doubles. -*/ + * @brief Dequantize two 64-bit integers to doubles. + */ __always_inline __m128d mm_dequantize_epi64_pd(const __m128i& input, const __m128d& scale) { const __m128d converted = convert_epi64_pd(input); return _mm_mul_pd(converted, scale); } /** -* @brief Dequantize eight 32-bit integers to floats. -*/ + * @brief Dequantize eight 32-bit integers to floats. + */ __always_inline __m256 mm256_dequantize_epi32(const __m256i& input, const __m256& scale) { const __m256 converted = _mm256_cvtepi32_ps(input); return _mm256_mul_ps(converted, scale); @@ -76,16 +72,16 @@ __always_inline __m256d convert_epi64_pd(__m256i v) { } /** -* @brief Dequantize four 64-bit integers to doubles. -*/ + * @brief Dequantize four 64-bit integers to doubles. + */ __always_inline __m256d mm256_dequantize_epi64_pd(const __m256i& input, const __m256d& scale) { const __m256d converted = convert_epi64_pd(input); return _mm256_mul_pd(converted, scale); } /** -* @brief Unpack four aligned 8-bit or 16-bit values directly from the input buffer. -*/ + * @brief Unpack four aligned 8-bit or 16-bit values directly from the input buffer. + */ template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) __m128i mm_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { @@ -107,8 +103,8 @@ __m128i mm_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { } /** -* @brief Unpack four values using the table-driven AVX2 shuffle path. -*/ + * @brief Unpack four values using the table-driven AVX2 shuffle path. + */ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) __m128i mm_unpack_epi32_avx2(const u8* __restrict__ input) { @@ -132,8 +128,8 @@ __m128i mm_unpack_epi32_avx2(const u8* __restrict__ input) { } /** -* @brief Unpack eight aligned 8-bit or 16-bit values directly from the input buffer. -*/ + * @brief Unpack eight aligned 8-bit or 16-bit values directly from the input buffer. + */ template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) __m256i mm256_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { @@ -155,8 +151,8 @@ __m256i mm256_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { } /** -* @brief Unpack eight values using the table-driven AVX2 shuffle path. -*/ + * @brief Unpack eight values using the table-driven AVX2 shuffle path. + */ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) __m256i mm256_unpack_epi32_avx2(const u8* __restrict__ input) { @@ -179,25 +175,24 @@ __m256i mm256_unpack_epi32_avx2(const u8* __restrict__ input) { return shifted; } -} // namespace internal +} // namespace internal /** -* @brief Decompress a single block to float using AVX2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* -* @param input pointer to the start of the compressed block. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed float values will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX2 support. -*/ + * @brief Decompress a single block to float using AVX2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * + * @param input pointer to the start of the compressed block. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed float values will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr) { +int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -211,14 +206,12 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f32 sc const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); _mm256_storeu_ps(output, dequantized); - input += BIT_WIDTH; + input += BIT_WIDTH; output += 8; } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback - (input, remaining); + const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); for (u32 i = 0; i < remaining; i++) { output[i] = internal::dequantize_epi32(tail_values[i], scale); } @@ -228,22 +221,21 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f32 sc } /** -* @brief Decompress a single block to double using AVX2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* -* @param input pointer to the start of the compressed block. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed double values will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX2 support. -*/ + * @brief Decompress a single block to double using AVX2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * + * @param input pointer to the start of the compressed block. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed double values will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr) { +int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -263,14 +255,12 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f64 sc _mm256_storeu_pd(output, dequantized1); _mm256_storeu_pd(output + 4, dequantized2); - input += BIT_WIDTH; + input += BIT_WIDTH; output += 8; } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback - (input, remaining); + const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); for (u32 i = 0; i < remaining; i++) { output[i] = internal::dequantize_epi64(tail_values[i], scale); } @@ -279,24 +269,22 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f64 sc } /** -* @brief Decompress multiple blocks to float using AVX2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* -* @param input pointer to the start of the compressed data. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed float values will be stored. -* @param blocks number of blocks to decompress. -* @return int status code (0 for success). -* -* @note This function requires AVX2 support. -*/ + * @brief Decompress multiple blocks to float using AVX2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * + * @param input pointer to the start of the compressed data. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed float values will be stored. + * @param blocks number of blocks to decompress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr, - const u32 blocks) { +int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -305,7 +293,7 @@ int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f32 s for (u32 block = 0; block < blocks; block++) { mm256_decompress_block_avx2(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -313,24 +301,22 @@ int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f32 s } /** -* @brief Decompress multiple blocks to double using AVX2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* -* @param input pointer to the start of the compressed data. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed float values will be stored. -* @param blocks number of blocks to decompress. -* @return int status code (0 for success). -* -* @note This function requires AVX2 support. -*/ + * @brief Decompress multiple blocks to double using AVX2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * + * @param input pointer to the start of the compressed data. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed float values will be stored. + * @param blocks number of blocks to decompress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr, - const u32 blocks) { +int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -339,12 +325,12 @@ int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f64 s for (u32 block = 0; block < blocks; block++) { mm256_decompress_block_avx2(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix +} // namespace pernix #endif // PERNIX_AVX2_DECOMPRESSION_H diff --git a/include/pernix/x86/avx2/avx2_tables.h b/include/pernix/x86/avx2/avx2_tables.h index 20f9cf4..4dd37cb 100644 --- a/include/pernix/x86/avx2/avx2_tables.h +++ b/include/pernix/x86/avx2/avx2_tables.h @@ -760,9 +760,7 @@ struct unpack_tables_avx2 { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" - __always_inline static __m256i get_permute() { - return _mm256_load_si256(reinterpret_cast(permute.data())); - } + __always_inline static __m256i get_permute() { return _mm256_load_si256(reinterpret_cast(permute.data())); } __always_inline static T get_shuffle() { if constexpr (std::is_same_v) { @@ -781,6 +779,6 @@ struct unpack_tables_avx2 { } #pragma GCC diagnostic pop }; -} // namespace pernix::internal +} // namespace pernix::internal #endif // PERNIX_AVX2_TABLES_H diff --git a/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h index c51be2c..39ef6c0 100644 --- a/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h +++ b/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h @@ -3,15 +3,14 @@ #include #include -#include -#include #include +#include +#include #include using namespace pernix::x86::internal; - namespace pernix { namespace internal { template @@ -19,8 +18,7 @@ template static __always_inline __m512i mm512_clamp_signed_epi32(__m512i input) { constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm512_min_epi32(_mm512_max_epi32(input, _mm512_set1_epi32(min_value)), - _mm512_set1_epi32(max_value)); + return _mm512_min_epi32(_mm512_max_epi32(input, _mm512_set1_epi32(min_value)), _mm512_set1_epi32(max_value)); } template @@ -28,8 +26,7 @@ template static __always_inline __m256i mm256_clamp_signed_epi32_avx512(__m256i input) { constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); - return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), - _mm256_set1_epi32(max_value)); + return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), _mm256_set1_epi32(max_value)); } template @@ -37,13 +34,12 @@ template static __always_inline __m512i mm512_clamp_signed_epi64(__m512i input) { constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); - return _mm512_min_epi64(_mm512_max_epi64(input, _mm512_set1_epi64(min_value)), - _mm512_set1_epi64(max_value)); + return _mm512_min_epi64(_mm512_max_epi64(input, _mm512_set1_epi64(min_value)), _mm512_set1_epi64(max_value)); } /** -* @brief Quantize sixteen float values to 32-bit integers. -*/ + * @brief Quantize sixteen float values to 32-bit integers. + */ static __always_inline __m512i mm512_quantize_ps_epi32(const __m512& input, const __m512& scale) { const __m512 scaled = _mm512_mul_ps(input, scale); @@ -66,8 +62,7 @@ static __always_inline __m512i make_m512i_from_2x256(const __m256i a, const __m2 return result; } -static __always_inline __m512i make_m512i_from_4x128(const __m128i a, const __m128i b, const __m128i c, - const __m128i d) { +static __always_inline __m512i make_m512i_from_4x128(const __m128i a, const __m128i b, const __m128i c, const __m128i d) { __m512i result = _mm512_castsi128_si512(a); result = _mm512_inserti64x2(result, b, 1); result = _mm512_inserti64x2(result, c, 2); @@ -75,8 +70,7 @@ static __always_inline __m512i make_m512i_from_4x128(const __m128i a, const __m1 return result; } -static __always_inline __m512i make_m512i_from_8x64(const __m128i a, const __m128i b, const __m128i c, - const __m128i d, const __m128i e, +static __always_inline __m512i make_m512i_from_8x64(const __m128i a, const __m128i b, const __m128i c, const __m128i d, const __m128i e, const __m128i f, const __m128i g, const __m128i h) { const __m128i ab = _mm_unpacklo_epi64(a, b); const __m128i cd = _mm_unpacklo_epi64(c, d); @@ -96,8 +90,7 @@ static __always_inline __m256i make_m256i_from_2x128(const __m128i a, const __m1 return result; } -static __always_inline __m256i make_m256i_from_4x64(const __m128i a, const __m128i b, const __m128i c, - const __m128i d) { +static __always_inline __m256i make_m256i_from_4x64(const __m128i a, const __m128i b, const __m128i c, const __m128i d) { const __m128i ab = _mm_unpacklo_epi64(a, b); const __m128i cd = _mm_unpacklo_epi64(c, d); @@ -108,15 +101,13 @@ static __always_inline __m256i make_m256i_from_4x64(const __m128i a, const __m12 template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_1to8(const f32* __restrict__ input, const f32 scale, - u8* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_1to8(const f32* __restrict__ input, const f32 scale, u8* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_64 = elements_per_block / 64; constexpr u32 iterations_32 = (elements_per_block % 64) / 32; constexpr u32 iterations_16 = (elements_per_block % 32) / 16; - constexpr u32 remaining_elements = - elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + constexpr u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; const __m512 scale_v = _mm512_set1_ps(scale); @@ -128,14 +119,10 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f32* __restrict__ const __m512 source3 = _mm512_loadu_ps(input + 32); const __m512 source4 = _mm512_loadu_ps(input + 48); - const __m512i quantized1 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source4, scale_v)); + const __m512i quantized1 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source4, scale_v)); const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); @@ -143,12 +130,11 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f32* __restrict__ const __m128i converted4 = _mm512_cvtepi32_epi8(quantized4); const __m512i packed = - m512::mm512_pack_epi8_avx512vbmi_1to8(make_m512i_from_4x128( - converted1, converted2, converted3, converted4)); + m512::mm512_pack_epi8_avx512vbmi_1to8(make_m512i_from_4x128(converted1, converted2, converted3, converted4)); mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - input += 64; + input += 64; output += 8 * BIT_WIDTH; } } @@ -157,19 +143,16 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f32* __restrict__ const __m512 source1 = _mm512_loadu_ps(input); const __m512 source2 = _mm512_loadu_ps(input + 16); - const __m512i quantized1 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source2, scale_v)); + const __m512i quantized1 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source2, scale_v)); const __m128i converted1 = _mm512_cvtepi32_epi8(quantized1); const __m128i converted2 = _mm512_cvtepi32_epi8(quantized2); - const __m256i packed = m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_2x128( - converted1, converted2)); + const __m256i packed = m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_2x128(converted1, converted2)); mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } @@ -181,7 +164,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f32* __restrict__ const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(converted); mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -199,15 +182,13 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f32* __restrict__ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_9to16(const f32* __restrict__ input, const f32 scale, - u8* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_9to16(const f32* __restrict__ input, const f32 scale, u8* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_32 = elements_per_block / 32; constexpr u32 iterations_16 = (elements_per_block % 32) / 16; constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = - elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; const __m512 scale_v = _mm512_set1_ps(scale); const __m256 scale_v256 = _mm256_set1_ps(scale); @@ -218,19 +199,16 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const f32* __restrict_ const __m512 source1 = _mm512_loadu_ps(input); const __m512 source2 = _mm512_loadu_ps(input + 16); - const __m512i quantized1 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source2, scale_v)); + const __m512i quantized1 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source2, scale_v)); const __m256i converted1 = _mm512_cvtepi32_epi16(quantized1); const __m256i converted2 = _mm512_cvtepi32_epi16(quantized2); - const __m512i packed = m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_2x256( - converted1, converted2)); + const __m512i packed = m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_2x256(converted1, converted2)); mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } } @@ -243,27 +221,25 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const f32* __restrict_ const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(converted); mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } if constexpr (iterations_8 > 0) { const __m256 source = _mm256_loadu_ps(input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512( - mm256_quantize_ps_epi32(source, scale_v256)); + const __m256i quantized = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); const __m128i converted = _mm256_cvtepi32_epi16(quantized); const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512( - mm256_quantize_ps_epi32(source, scale_v256)); + const __m256i quantized = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); const __m128i converted = _mm256_cvtepi32_epi16(quantized); const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); @@ -275,8 +251,7 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const f32* __restrict_ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_17to24(const f32* __restrict__ input, const f32 scale, - u8* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_17to24(const f32* __restrict__ input, const f32 scale, u8* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_16 = elements_per_block / 16; @@ -290,33 +265,30 @@ __always_inline int mm512_compress_block_avx512vbmi_17to24(const f32* __restrict #pragma GCC unroll 2 for (u32 i = 0; i < iterations_16; ++i) { const __m512 source = _mm512_loadu_ps(input); - const __m512i packed_input = mm512_clamp_signed_epi32( - mm512_quantize_ps_epi32(source, scale_v)); + const __m512i packed_input = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(packed_input); mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } } if constexpr (iterations_8 > 0) { const __m256 source = _mm256_loadu_ps(input); - const __m256i packed_input = mm256_clamp_signed_epi32_avx512( - mm256_quantize_ps_epi32(source, scale_v256)); + const __m256i packed_input = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); - const __m256i packed_input = mm256_clamp_signed_epi32_avx512( - mm256_quantize_ps_epi32(source, scale_v256)); - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); + const __m256i packed_input = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); } @@ -326,15 +298,13 @@ __always_inline int mm512_compress_block_avx512vbmi_17to24(const f32* __restrict template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ input, const f64 scale, - u8* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ input, const f64 scale, u8* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_64 = elements_per_block / 64; constexpr u32 iterations_32 = (elements_per_block % 64) / 32; constexpr u32 iterations_16 = (elements_per_block % 32) / 16; - constexpr u32 remaining_elements = - elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + constexpr u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; const __m512d scale_v = _mm512_set1_pd(scale); @@ -350,22 +320,14 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ const __m512d source7 = _mm512_loadu_pd(input + 48); const __m512d source8 = _mm512_loadu_pd(input + 56); - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source4, scale_v)); - const __m512i quantized5 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source5, scale_v)); - const __m512i quantized6 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source6, scale_v)); - const __m512i quantized7 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source7, scale_v)); - const __m512i quantized8 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source8, scale_v)); + const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source4, scale_v)); + const __m512i quantized5 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source5, scale_v)); + const __m512i quantized6 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source6, scale_v)); + const __m512i quantized7 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source7, scale_v)); + const __m512i quantized8 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source8, scale_v)); const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); @@ -377,12 +339,11 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ const __m128i converted8 = _mm512_cvtepi64_epi8(quantized8); const __m512i packed = m512::mm512_pack_epi8_avx512vbmi_1to8( - make_m512i_from_8x64(converted1, converted2, converted3, converted4, - converted5, converted6, converted7, converted8)); + make_m512i_from_8x64(converted1, converted2, converted3, converted4, converted5, converted6, converted7, converted8)); mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - input += 64; + input += 64; output += 8 * BIT_WIDTH; } } @@ -393,14 +354,10 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ const __m512d source3 = _mm512_loadu_pd(input + 16); const __m512d source4 = _mm512_loadu_pd(input + 24); - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source4, scale_v)); + const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source4, scale_v)); const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); @@ -408,32 +365,28 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ const __m128i converted4 = _mm512_cvtepi64_epi8(quantized4); const __m256i packed = - m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_4x64( - converted1, converted2, converted3, converted4)); + m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_4x64(converted1, converted2, converted3, converted4)); mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } if constexpr (iterations_16 > 0) { const __m512d source1 = _mm512_loadu_pd(input); const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(_mm_unpacklo_epi64( - converted1, converted2)); + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(_mm_unpacklo_epi64(converted1, converted2)); mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -441,20 +394,15 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ constexpr u32 source1_elements = remaining_elements > 8 ? 8 : remaining_elements; constexpr u32 source2_elements = remaining_elements > 8 ? remaining_elements - 8 : 0; - const __m512d source1 = mm512_loadu_elements_pd(source1_elements, input); - const __m512d source2 = source2_elements > 0 - ? mm512_loadu_elements_pd(source2_elements, input + 8) - : _mm512_setzero_pd(); - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); + const __m512d source1 = mm512_loadu_elements_pd(source1_elements, input); + const __m512d source2 = source2_elements > 0 ? mm512_loadu_elements_pd(source2_elements, input + 8) : _mm512_setzero_pd(); + const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); const __m128i converted1 = _mm512_cvtepi64_epi8(quantized1); const __m128i converted2 = _mm512_cvtepi64_epi8(quantized2); - const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(_mm_unpacklo_epi64( - converted1, converted2)); + const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(_mm_unpacklo_epi64(converted1, converted2)); mm_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); } @@ -464,15 +412,13 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_9to16(const f64* __restrict__ input, const f64 scale, - u8* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_9to16(const f64* __restrict__ input, const f64 scale, u8* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_32 = elements_per_block / 32; constexpr u32 iterations_16 = (elements_per_block % 32) / 16; constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = - elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; const __m512d scale_v = _mm512_set1_pd(scale); @@ -484,14 +430,10 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const f64* __restrict_ const __m512d source3 = _mm512_loadu_pd(input + 16); const __m512d source4 = _mm512_loadu_pd(input + 24); - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); - const __m512i quantized3 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source3, scale_v)); - const __m512i quantized4 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source4, scale_v)); + const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized3 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source3, scale_v)); + const __m512i quantized4 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source4, scale_v)); const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); @@ -499,12 +441,11 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const f64* __restrict_ const __m128i converted4 = _mm512_cvtepi64_epi16(quantized4); const __m512i packed = - m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_4x128( - converted1, converted2, converted3, converted4)); + m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_4x128(converted1, converted2, converted3, converted4)); mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } } @@ -512,20 +453,17 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const f64* __restrict_ if constexpr (iterations_16 > 0) { const __m512d source1 = _mm512_loadu_pd(input); const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m512i quantized1 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source1, scale_v)); - const __m512i quantized2 = mm512_clamp_signed_epi64( - mm512_quantize_pd_epi64(source2, scale_v)); + const __m512i quantized1 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source1, scale_v)); + const __m512i quantized2 = mm512_clamp_signed_epi64(mm512_quantize_pd_epi64(source2, scale_v)); const __m128i converted1 = _mm512_cvtepi64_epi16(quantized1); const __m128i converted2 = _mm512_cvtepi64_epi16(quantized2); - const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(make_m256i_from_2x128( - converted1, converted2)); + const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(make_m256i_from_2x128(converted1, converted2)); mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -537,7 +475,7 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const f64* __restrict_ const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -555,8 +493,7 @@ __always_inline int mm512_compress_block_avx512vbmi_9to16(const f64* __restrict_ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_17to24(const f64* __restrict__ input, const f64 scale, - u8* __restrict__ output) { +__always_inline int mm512_compress_block_avx512vbmi_17to24(const f64* __restrict__ input, const f64 scale, u8* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_16 = elements_per_block / 16; @@ -571,61 +508,55 @@ __always_inline int mm512_compress_block_avx512vbmi_17to24(const f64* __restrict const __m512d source1 = _mm512_loadu_pd(input); const __m512d source2 = _mm512_loadu_pd(input + 8); - const __m256i quantized1 = mm256_clamp_signed_epi32_avx512( - mm512_quantize_pd_epi32(source1, scale_v)); - const __m256i quantized2 = mm256_clamp_signed_epi32_avx512( - mm512_quantize_pd_epi32(source2, scale_v)); + const __m256i quantized1 = mm256_clamp_signed_epi32_avx512(mm512_quantize_pd_epi32(source1, scale_v)); + const __m256i quantized2 = mm256_clamp_signed_epi32_avx512(mm512_quantize_pd_epi32(source2, scale_v)); - const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24( - make_m512i_from_2x256(quantized1, quantized2)); + const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(make_m512i_from_2x256(quantized1, quantized2)); mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } } if constexpr (iterations_8 > 0) { const __m512d source = _mm512_loadu_pd(input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512( - mm512_quantize_pd_epi32(source, scale_v)); + const __m256i quantized = mm256_clamp_signed_epi32_avx512(mm512_quantize_pd_epi32(source, scale_v)); const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m512d source = mm512_loadu_elements_pd(remaining_elements, input); - const __m256i quantized = mm256_clamp_signed_epi32_avx512( - mm512_quantize_pd_epi32(source, scale_v)); - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); + const __m256i quantized = mm256_clamp_signed_epi32_avx512(mm512_quantize_pd_epi32(source, scale_v)); + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); } return 0; } -} // namespace internal +} // namespace internal /** -* @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* -* @param input pointer to the start of the input float values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX-512 and AVX-512-VBMI support. -*/ + * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * + * @param input pointer to the start of the input float values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX-512 and AVX-512-VBMI support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr) { +int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -641,20 +572,19 @@ int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f3 } /** -* @brief Compress a single block of double values using AVX-512 and AVX-512-VBMI instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @param input pointer to the start of the input double values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @return int status code. -* -* @note This overload is declared for parity with the float path. -*/ + * @brief Compress a single block of double values using AVX-512 and AVX-512-VBMI instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @param input pointer to the start of the input double values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @return int status code. + * + * @note This overload is declared for parity with the float path. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr) { +int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -670,23 +600,21 @@ int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f6 } /** -* @brief Compress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* -* @param input pointer to the start of the input float values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @param blocks number of 512-bit blocks to compress. -* @return int status code (0 for success). -* -* @note This function requires AVX-512 and AVX-512-VBMI support. -*/ + * @brief Compress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * + * @param input pointer to the start of the input float values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @param blocks number of 512-bit blocks to compress. + * @return int status code (0 for success). + * + * @note This function requires AVX-512 and AVX-512-VBMI support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr, - const u32 blocks) { +int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -695,7 +623,7 @@ int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f for (u32 block = 0; block < blocks; ++block) { mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } @@ -703,20 +631,18 @@ int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f } /** -* @brief Compress multiple blocks of double values using AVX-512 and AVX-512-VBMI instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @param input pointer to the start of the input double values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @param blocks number of blocks to compress. -* @return int status code. -*/ + * @brief Compress multiple blocks of double values using AVX-512 and AVX-512-VBMI instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @param input pointer to the start of the input double values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @param blocks number of blocks to compress. + * @return int status code. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr, - const u32 blocks) { +int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -725,12 +651,12 @@ int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f for (u32 block = 0; block < blocks; ++block) { mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } return 0; } -} // namespace pernix +} // namespace pernix #endif // PERNIX_AVX512VBMI_COMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h index 7969b80..ba1c6c9 100644 --- a/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h +++ b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h @@ -9,14 +9,13 @@ #include - using namespace pernix::x86::internal; namespace pernix { namespace internal { /** -* @brief Dequantize sixteen integer values to floats. -*/ + * @brief Dequantize sixteen integer values to floats. + */ __always_inline __m512 mm512_dequantize_epi32(const __m512i& input, const __m512& scale) { const __m512 converted = _mm512_cvtepi32_ps(input); return _mm512_mul_ps(converted, scale); @@ -29,15 +28,13 @@ __always_inline __m512d mm512_dequantize_epi64(const __m512i& input, const __m51 template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict__ input, const f32 scale, - f32* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const u32 iterations_64 = elements_per_block / 64; const u32 iterations_32 = (elements_per_block % 64) / 32; const u32 iterations_16 = (elements_per_block % 32) / 16; - const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - - iterations_16 * 16; + const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; const __m512 scale_v = _mm512_set1_ps(scale); @@ -45,9 +42,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ #pragma GCC unroll 8 for (u32 i = 0; i < iterations_64; ++i) { const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8 - (source); + const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8(source); const __m512i converted1 = _mm512_cvtepi8_epi32(_mm512_castsi512_si128(unpacked)); const __m512i converted2 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 1)); @@ -65,15 +60,13 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ _mm512_storeu_ps(output + 48, dequantized4); output += 64; - input += 8 * BIT_WIDTH; + input += 8 * BIT_WIDTH; } } if constexpr (iterations_32 > 0) { const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8 - (source); + const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8(source); const __m512i converted1 = _mm512_cvtepi8_epi32(_mm256_castsi256_si128(unpacked)); const __m512i converted2 = _mm512_cvtepi8_epi32(_mm256_extracti128_si256(unpacked, 1)); @@ -85,14 +78,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ _mm512_storeu_ps(output + 16, dequantized2); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } if constexpr (iterations_16 > 0) { const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 - (source); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); const __m512i converted = _mm512_cvtepi8_epi32(unpacked); @@ -101,14 +92,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 - (source); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); const __m512i converted = _mm512_cvtepi8_epi32(unpacked); @@ -122,15 +111,13 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict__ input, const f64 scale, - f64* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; const u32 iterations_64 = elements_per_block / 64; const u32 iterations_32 = (elements_per_block % 64) / 32; const u32 iterations_16 = (elements_per_block % 32) / 16; - const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - - iterations_16 * 16; + const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; const __m512d scale_v = _mm512_set1_pd(scale); @@ -138,9 +125,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ #pragma GCC unroll 8 for (u32 i = 0; i < iterations_64; ++i) { const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8 - (source); + const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8(source); const __m128i extracted1 = _mm512_castsi512_si128(unpacked); const __m128i extracted2 = _mm512_extracti64x2_epi64(unpacked, 1); @@ -175,14 +160,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ _mm512_storeu_pd(output + 56, dequantized8); output += 64; - input += 8 * BIT_WIDTH; + input += 8 * BIT_WIDTH; } if constexpr (iterations_32 > 0) { const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8 - (source); + const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8(source); const __m128i extracted1 = _mm256_castsi256_si128(unpacked); const __m128i extracted2 = _mm256_extracti64x2_epi64(unpacked, 1); @@ -203,14 +186,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ _mm512_storeu_pd(output + 24, dequantized4); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } if constexpr (iterations_16 > 0) { const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 - (source); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); @@ -222,14 +203,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8 - (source); + const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); @@ -249,15 +228,13 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict__ input, const f32 scale, - f32* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_32 = elements_per_block / 32; constexpr u32 iterations_16 = (elements_per_block % 32) / 16; constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = - elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; const __m512 scale_v = _mm512_set1_ps(scale); const __m256 scale_v256 = _mm256_set1_ps(scale); @@ -266,9 +243,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict #pragma GCC unroll 4 for (u32 i = 0; i < iterations_32; ++i) { const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16 - (source); + const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16(source); const __m512i converted1 = _mm512_cvtepi16_epi32(_mm512_castsi512_si256(unpacked)); const __m512i converted2 = _mm512_cvtepi16_epi32(_mm512_extracti32x8_epi32(unpacked, 1)); @@ -280,15 +255,13 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict _mm512_storeu_ps(output + 16, dequantized2); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } } if constexpr (iterations_16 > 0) { const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16 - (source); + const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16(source); const __m512i converted = _mm512_cvtepi16_epi32(unpacked); const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); @@ -296,14 +269,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (iterations_8 > 0) { const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 - (source); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); const __m256i converted = _mm256_cvtepi16_epi32(unpacked); const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); @@ -311,14 +282,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict _mm256_storeu_ps(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 - (source); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); const __m256i converted = _mm256_cvtepi16_epi32(unpacked); const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); @@ -331,15 +300,13 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict__ input, const f64 scale, - f64* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_32 = elements_per_block / 32; constexpr u32 iterations_16 = (elements_per_block % 32) / 16; constexpr u32 iterations_8 = (elements_per_block % 16) / 8; - constexpr u32 remaining_elements = - elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; const __m512d scale_v = _mm512_set1_pd(scale); @@ -347,9 +314,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict #pragma GCC unroll 4 for (u32 i = 0; i < iterations_32; ++i) { const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16 - (source); + const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16(source); const __m512i converted1 = _mm512_cvtepi16_epi64(_mm512_castsi512_si128(unpacked)); const __m512i converted2 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 1)); @@ -367,15 +332,13 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict _mm512_storeu_pd(output + 24, dequantized4); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } } if constexpr (iterations_16 > 0) { const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16 - (source); + const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16(source); const __m512i converted1 = _mm512_cvtepi16_epi64(_mm256_castsi256_si128(unpacked)); const __m512i converted2 = _mm512_cvtepi16_epi64(_mm256_extracti64x2_epi64(unpacked, 1)); @@ -387,14 +350,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (iterations_8 > 0) { const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 - (source); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); const __m512i converted = _mm512_cvtepi16_epi64(unpacked); @@ -403,14 +364,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict _mm512_storeu_pd(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16 - (source); + const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); const __m512i converted = _mm512_cvtepi16_epi64(unpacked); @@ -424,8 +383,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restrict__ input, const f32 scale, - f32* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_16 = elements_per_block / 16; @@ -439,38 +397,32 @@ __always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restric #pragma GCC unroll 2 for (u32 i = 0; i < iterations_16; ++i) { const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24 - (source); + const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24(source); const __m512 dequantized = mm512_dequantize_epi32(unpacked, scale_v); _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } } if constexpr (iterations_8 > 0) { const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 - (source); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24(source); const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); _mm256_storeu_ps(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 - (source); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24(source); const __m256 dequantized = mm256_dequantize_epi32(unpacked, scale_v256); @@ -482,8 +434,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restric template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restrict__ input, const f64 scale, - f64* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; constexpr u32 iterations_16 = elements_per_block / 16; @@ -496,9 +447,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restric #pragma GCC unroll 2 for (u32 i = 0; i < iterations_16; ++i) { const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); - const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24 - (source); + const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24(source); const __m512i converted1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(unpacked)); const __m512i converted2 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64(unpacked, 1)); @@ -510,15 +459,13 @@ __always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restric _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } } if constexpr (iterations_8 > 0) { const __m256i source = mm256_loadu_elements_epi8(BIT_WIDTH, input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 - (source); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24(source); const __m512i converted = _mm512_cvtepi32_epi64(unpacked); @@ -527,14 +474,12 @@ __always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restric _mm512_storeu_pd(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m256i source = mm256_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); - const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24 - (source); + const __m256i unpacked = m256::mm256_unpack_epi32_avx512vbmi_17to24(source); const __m512i converted = _mm512_cvtepi32_epi64(unpacked); @@ -545,89 +490,80 @@ __always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restric return 0; } -} // namespace internal +} // namespace internal /** -* @brief Decompress a single 512\-bit block using AVX-512 and AVX-512-VBMI instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* -* @param input pointer to the start of the compressed block. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed float values will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX-512 and AVX-512-VBMI support. -*/ + * @brief Decompress a single 512\-bit block using AVX-512 and AVX-512-VBMI instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * + * @param input pointer to the start of the compressed block. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed float values will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX-512 and AVX-512-VBMI support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr) { +__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::mm512_decompress_block_avx512vbmi_1to8( - input, scale, output); + return internal::mm512_decompress_block_avx512vbmi_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::mm512_decompress_block_avx512vbmi_9to16( - input, scale, output); + return internal::mm512_decompress_block_avx512vbmi_9to16(input, scale, output); } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::mm512_decompress_block_avx512vbmi_17to24( - input, scale, output); + return internal::mm512_decompress_block_avx512vbmi_17to24(input, scale, output); } return 0; } /** -* @brief Decompress a single block to double values using AVX-512 and AVX-512-VBMI instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @param input pointer to the start of the compressed block. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed double values will be stored. -* @return int status code. -*/ + * @brief Decompress a single block to double values using AVX-512 and AVX-512-VBMI instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * @param input pointer to the start of the compressed block. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed double values will be stored. + * @return int status code. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr) { +__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::mm512_decompress_block_avx512vbmi_1to8( - input, scale, output); + return internal::mm512_decompress_block_avx512vbmi_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::mm512_decompress_block_avx512vbmi_9to16( - input, scale, output); + return internal::mm512_decompress_block_avx512vbmi_9to16(input, scale, output); } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { - return internal::mm512_decompress_block_avx512vbmi_17to24( - input, scale, output); + return internal::mm512_decompress_block_avx512vbmi_17to24(input, scale, output); } return 0; } /** -* @brief Decompress multiple 512\-bit blocks using AVX-512 and AVX-512-VBMI instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* -* @param input pointer to the start of the compressed data. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed float values will be stored. -* @param blocks number of 512-bit blocks to decompress. -* @return int status code (0 for success). -* -* @note This function requires AVX-512 and AVX-512-VBMI support. -*/ + * @brief Decompress multiple 512\-bit blocks using AVX-512 and AVX-512-VBMI instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * + * @param input pointer to the start of the compressed data. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed float values will be stored. + * @param blocks number of 512-bit blocks to decompress. + * @return int status code (0 for success). + * + * @note This function requires AVX-512 and AVX-512-VBMI support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr, +int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -637,7 +573,7 @@ int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const for (u32 block = 0; block < blocks; ++block) { mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -645,20 +581,19 @@ int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const } /** -* @brief Decompress multiple blocks to double values using AVX-512 and AVX-512-VBMI instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @param input pointer to the start of the compressed data. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed double values will be stored. -* @param blocks number of blocks to decompress. -* @return int status code. -*/ + * @brief Decompress multiple blocks to double values using AVX-512 and AVX-512-VBMI instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * @param input pointer to the start of the compressed data. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed double values will be stored. + * @param blocks number of blocks to decompress. + * @return int status code. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr, +int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -668,11 +603,11 @@ int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const for (u32 block = 0; block < blocks; ++block) { mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix +} // namespace pernix #endif // PERNIX_AVX512VBMI_DECOMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/compat.h b/include/pernix/x86/avx512vbmi/compat.h index 81f58e8..05d346e 100644 --- a/include/pernix/x86/avx512vbmi/compat.h +++ b/include/pernix/x86/avx512vbmi/compat.h @@ -2,11 +2,11 @@ #define PERNIX_AVX512_COMPAT_H #include + #include #include #include - namespace pernix::internal { static __always_inline __mmask8 element_mask8(const u32 e) { return static_cast<__mmask8>(e >= 8 ? 0xFFu : ((1u << e) - 1u)); @@ -383,6 +383,6 @@ static __always_inline void mm_storeu_elements_pd(void* mem_addr, const u32 e, c _mm_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif } -} +} // namespace pernix::internal -#endif //PERNIX_AVX512_COMPAT_H +#endif // PERNIX_AVX512_COMPAT_H diff --git a/include/pernix/x86/avx512vbmi/packing.h b/include/pernix/x86/avx512vbmi/packing.h index ebdd066..158d7d1 100644 --- a/include/pernix/x86/avx512vbmi/packing.h +++ b/include/pernix/x86/avx512vbmi/packing.h @@ -1,15 +1,14 @@ #ifndef PERNIX_AVX512VBMI_PACKING_H #define PERNIX_AVX512VBMI_PACKING_H -#include #include - +#include namespace pernix::internal { namespace m128 { /** -* @brief Pack 8 16-bit values for bit widths 9 through 16 using VBMI. -*/ + * @brief Pack 8 16-bit values for bit widths 9 through 16 using VBMI. + */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { @@ -31,12 +30,9 @@ __always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { } else { const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - const __m128i permuted1 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask1), - tables::get_permute1(), masked); - const __m128i permuted2 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask2), - tables::get_permute2(), masked); - const __m128i permuted3 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask3), - tables::get_permute3(), masked); + const __m128i permuted1 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask1), tables::get_permute1(), masked); + const __m128i permuted2 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask2), tables::get_permute2(), masked); + const __m128i permuted3 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask3), tables::get_permute3(), masked); const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); @@ -48,8 +44,8 @@ __always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { } /** -* @brief Pack 16 8-bit values for bit widths 1 through 8 using VBMI. -*/ + * @brief Pack 16 8-bit values for bit widths 1 through 8 using VBMI. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { @@ -93,8 +89,8 @@ __always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { } /** -* @brief Pack 4 32-bit values for bit widths 17 through 24 using VBMI. -*/ + * @brief Pack 4 32-bit values for bit widths 17 through 24 using VBMI. + */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i& input) { @@ -113,12 +109,12 @@ __always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i& input) { return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); } -} // namespace m128 +} // namespace m128 namespace m256 { /** -* @brief Pack 16 16-bit values for bit widths 9 through 16 using VBMI. -*/ + * @brief Pack 16 16-bit values for bit widths 9 through 16 using VBMI. + */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) { @@ -140,12 +136,9 @@ __always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) } else { const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask1), - tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask2), - tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask3), - tables::get_permute3(), masked); + const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask1), tables::get_permute1(), masked); + const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask2), tables::get_permute2(), masked); + const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask3), tables::get_permute3(), masked); const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); @@ -157,8 +150,8 @@ __always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) } /** -* @brief Pack 32 8-bit values for bit widths 1 through 8 using VBMI. -*/ + * @brief Pack 32 8-bit values for bit widths 1 through 8 using VBMI. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { @@ -169,8 +162,7 @@ __always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { const __m256i masked = _mm256_and_si256(input, maskv); if constexpr (BIT_WIDTH == 1) { - return _mm256_set1_epi32( - static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); + return _mm256_set1_epi32(static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); } else if constexpr (BIT_WIDTH == 2) { const __m256i shifted = _mm256_srli_epi16(masked, 6); const __m256i combined = _mm256_or_si256(masked, shifted); @@ -186,8 +178,7 @@ __always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { const __m256i pair6 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 5)); const __m256i packed12 = _mm256_or_si256(pair6, _mm256_srli_epi32(pair6, 10)); - return m256::mm256_pack_epi16_avx512vbmi_9to16<12>( - _mm256_castsi128_si256(_mm256_cvtepi32_epi16(packed12))); + return m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm256_castsi128_si256(_mm256_cvtepi32_epi16(packed12))); } else if constexpr (BIT_WIDTH == 4) { const __m256i shifted = _mm256_srli_epi16(masked, 4); const __m256i combined = _mm256_or_si256(masked, shifted); @@ -204,8 +195,8 @@ __always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { } /** -* @brief Pack 8 32-bit values for bit widths 17 through 24 using VBMI. -*/ + * @brief Pack 8 32-bit values for bit widths 17 through 24 using VBMI. + */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i& input) { @@ -224,12 +215,12 @@ __always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i& input) return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); } -} // namespace m256 +} // namespace m256 namespace m512 { /** -* @brief Pack 32 16-bit values for bit widths 9 through 16 using VBMI. -*/ + * @brief Pack 32 16-bit values for bit widths 9 through 16 using VBMI. + */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) __always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) { @@ -251,12 +242,9 @@ __always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) } else { const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask1), - tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask2), - tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask3), - tables::get_permute3(), masked); + const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask1), tables::get_permute1(), masked); + const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask2), tables::get_permute2(), masked); + const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask3), tables::get_permute3(), masked); const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); @@ -268,8 +256,8 @@ __always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) } /** -* @brief Pack 64 8-bit values for bit widths 1 through 8 using VBMI. -*/ + * @brief Pack 64 8-bit values for bit widths 1 through 8 using VBMI. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) __always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { @@ -280,8 +268,7 @@ __always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { const __m512i masked = _mm512_and_si512(input, maskv); if constexpr (BIT_WIDTH == 1) { - return _mm512_set1_epi64( - static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); + return _mm512_set1_epi64(static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); } else if constexpr (BIT_WIDTH == 2) { const __m512i shifted = _mm512_srli_epi16(masked, 6); const __m512i combined = _mm512_or_si512(masked, shifted); @@ -297,8 +284,7 @@ __always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { const __m512i pair6 = _mm512_or_si512(even, _mm512_srli_epi16(odd, 5)); const __m512i packed12 = _mm512_or_si512(pair6, _mm512_srli_epi32(pair6, 10)); - return _mm512_castsi256_si512( - m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm512_cvtepi32_epi16(packed12))); + return _mm512_castsi256_si512(m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm512_cvtepi32_epi16(packed12))); } else if constexpr (BIT_WIDTH == 4) { const __m512i shifted = _mm512_srli_epi16(masked, 4); const __m512i combined = _mm512_or_si512(masked, shifted); @@ -315,8 +301,8 @@ __always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { } /** -* @brief Pack 16 32-bit values for bit widths 17 through 24 using VBMI. -*/ + * @brief Pack 16 32-bit values for bit widths 17 through 24 using VBMI. + */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i& input) { @@ -335,7 +321,7 @@ __always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i& input) return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); } -} // namespace m512 -} // namespace pernix::internal +} // namespace m512 +} // namespace pernix::internal #endif // PERNIX_AVX512VBMI_PACKING_H diff --git a/include/pernix/x86/avx512vbmi/tables.h b/include/pernix/x86/avx512vbmi/tables.h index d63a47a..d708e10 100644 --- a/include/pernix/x86/avx512vbmi/tables.h +++ b/include/pernix/x86/avx512vbmi/tables.h @@ -607,26 +607,17 @@ struct pack_tables_avx512_24 { return values; } - alignas(64) static constexpr auto permute1 = make_table([](const word_plan& p) { - return p.left_index1; - }); + alignas(64) static constexpr auto permute1 = make_table([](const word_plan& p) { return p.left_index1; }); - alignas(64) static constexpr auto permute2 = make_table([](const word_plan& p) { - return p.left_index2; - }); + alignas(64) static constexpr auto permute2 = make_table([](const word_plan& p) { return p.left_index2; }); - alignas(64) static constexpr auto permute3 = make_table([](const word_plan& p) { - return p.right_index; - }); + alignas(64) static constexpr auto permute3 = make_table([](const word_plan& p) { return p.right_index; }); - alignas(64) static constexpr auto shift1 = make_table( - [](const word_plan& p) { return p.left_shift1; }); + alignas(64) static constexpr auto shift1 = make_table([](const word_plan& p) { return p.left_shift1; }); - alignas(64) static constexpr auto shift2 = make_table( - [](const word_plan& p) { return p.left_shift2; }); + alignas(64) static constexpr auto shift2 = make_table([](const word_plan& p) { return p.left_shift2; }); - alignas(64) static constexpr auto shift3 = make_table( - [](const word_plan& p) { return p.right_shift; }); + alignas(64) static constexpr auto shift3 = make_table([](const word_plan& p) { return p.right_shift; }); public: static __always_inline Vec get_permute1() { return load_table(permute1); } @@ -824,6 +815,6 @@ struct unpack_tables_avx512_24 { static __always_inline Vec get_permute() { return load_table(permute); } static __always_inline Vec get_shift() { return load_table(shift); } }; -} // namespace pernix::internal +} // namespace pernix::internal #endif // PERNIX_AVX512VBMI_TABLES_H diff --git a/include/pernix/x86/avx512vbmi/unpacking.h b/include/pernix/x86/avx512vbmi/unpacking.h index e021819..06c29b3 100644 --- a/include/pernix/x86/avx512vbmi/unpacking.h +++ b/include/pernix/x86/avx512vbmi/unpacking.h @@ -1,14 +1,13 @@ #ifndef PERNIX_AVX512VBMI_UNPACKING_H #define PERNIX_AVX512VBMI_UNPACKING_H -#include #include +#include namespace pernix::internal { namespace m128 { constexpr __mmask16 kAlternateByteMask16 = 0xAAAAULL; - __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i count) { const __m128i mask = _mm_set1_epi16(0x00ff); const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); @@ -174,7 +173,7 @@ __always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i& input) return shifted; } -} // namespace m128 +} // namespace m128 namespace m256 { constexpr __mmask32 kAlternateByteMask32 = 0xAAAAAAAAULL; @@ -344,7 +343,7 @@ __always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i& inpu return shifted; } -} // namespace m256 +} // namespace m256 namespace m512 { constexpr __mmask64 kAlternateByteMask64 = 0xAAAAAAAAAAAAAAAAULL; @@ -513,7 +512,7 @@ __always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i& inpu return shifted; } -} // namespace m512 -} // namespace pernix::internal +} // namespace m512 +} // namespace pernix::internal #endif // PERNIX_AVX512VBMI_UNPACKING_H diff --git a/include/pernix/x86/bmi2/bmi2_compression.h b/include/pernix/x86/bmi2/bmi2_compression.h index b1f48da..5343f28 100644 --- a/include/pernix/x86/bmi2/bmi2_compression.h +++ b/include/pernix/x86/bmi2/bmi2_compression.h @@ -1,23 +1,22 @@ #ifndef PERNIX_BMI2_COMPRESSION_H #define PERNIX_BMI2_COMPRESSION_H -#include #include #include +#include #include #include #include - namespace pernix { namespace internal { /** -* @brief Build the masks and shift constants used by the BMI2 packers. -* -* @tparam BIT_WIDTH bit width per packed value. -* @return std::tuple mask tuple used by the BMI2 helpers. -*/ + * @brief Build the masks and shift constants used by the BMI2 packers. + * + * @tparam BIT_WIDTH bit width per packed value. + * @return std::tuple mask tuple used by the BMI2 helpers. + */ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) static constexpr std::tuple pack_avx2_bmi2_constants() { @@ -43,12 +42,12 @@ static constexpr std::tuple pack_avx2_bmi2_constants() { } /** -* @brief Pack four 32-bit values with BMI2 extract instructions. -* -* @tparam BIT_WIDTH bit width per packed value. -* @param input SIMD register containing four quantized values. -* @return __m128i packed bitstream in the low bytes of the result. -*/ + * @brief Pack four 32-bit values with BMI2 extract instructions. + * + * @tparam BIT_WIDTH bit width per packed value. + * @param input SIMD register containing four quantized values. + * @return __m128i packed bitstream in the low bytes of the result. + */ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { @@ -66,7 +65,7 @@ static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { const u64 temp_combined = _pext_u64(_mm_extract_epi64(input, 1), pext_mask); values[1] = temp_combined >> shift2; - values[0] |= (temp_combined << shift1); + values[0] |= (temp_combined << shift1); const __m128i result = _mm_set_epi64x(static_cast(values[1]), static_cast(values[0])); return result; @@ -74,12 +73,12 @@ static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { } /** -* @brief Pack eight 32-bit values with BMI2 extract instructions. -* -* @tparam BIT_WIDTH bit width per packed value. -* @param input SIMD register containing eight quantized values. -* @return __m256i packed bitstream in the low bytes of the result. -*/ + * @brief Pack eight 32-bit values with BMI2 extract instructions. + * + * @tparam BIT_WIDTH bit width per packed value. + * @param input SIMD register containing eight quantized values. + * @return __m256i packed bitstream in the low bytes of the result. + */ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { @@ -107,7 +106,7 @@ static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { const __m256i result = _mm256_setr_epi64x(values[0], values[1], 0, 0); return result; } else { - constexpr u32 chunk_bits = BIT_WIDTH * 2; // bits extracted per 64-bit lane + constexpr u32 chunk_bits = BIT_WIDTH * 2; // bits extracted per 64-bit lane static_assert(chunk_bits < 64); constexpr u64 chunk_mask = (chunk_bits == 64) ? ~u64{0} : ((u64{1} << chunk_bits) - 1); @@ -122,8 +121,8 @@ static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { u64 out2 = 0; auto append_bits = [&](u64 value, u32 bit_offset) { - const u32 word = bit_offset >> 6; // / 64 - const u32 off = bit_offset & 63; // % 64 + const u32 word = bit_offset >> 6; // / 64 + const u32 off = bit_offset & 63; // % 64 if (word == 0) { out0 |= value << off; @@ -145,28 +144,26 @@ static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { append_bits(x2, 2 * chunk_bits); append_bits(x3, 3 * chunk_bits); - return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), - static_cast(out2), 0); + return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), static_cast(out2), 0); } } -} // namespace internal +} // namespace internal /** -* @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* -* @param input pointer to the start of the input float values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX2 and BMI2 support. -*/ + * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * + * @param input pointer to the start of the input float values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr) { +int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -184,7 +181,7 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scal const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); std::memcpy(output, &packed, BIT_WIDTH); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -192,9 +189,7 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scal std::vector block_values(remaining); #pragma GCC unroll 8 for (u32 i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized( - internal::quantize_ps_epi32(input[i], scale))); + block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], scale))); } internal::pack_epi32_fallback(block_values, output); @@ -204,20 +199,19 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scal } /** -* @brief Compress a single block of double values using AVX2 and BMI2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @param input pointer to the start of the input double values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX2 and BMI2 support. -*/ + * @brief Compress a single block of double values using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @param input pointer to the start of the input double values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr) { +int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -236,10 +230,9 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scal const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); __m256i combined = _mm256_castsi128_si256(quantized1); combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_bmi2( - internal::mm256_clamp_signed_epi32(combined)); + const __m256i packed = internal::mm256_pack_epi32_bmi2(internal::mm256_clamp_signed_epi32(combined)); std::memcpy(output, &packed, BIT_WIDTH); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -247,9 +240,7 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scal std::vector block_values(remaining); #pragma GCC unroll 8 for (u32 i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized( - internal::quantize_pd_epi64(input[i], scale))); + block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], scale))); } internal::pack_epi32_fallback(block_values, output); @@ -258,22 +249,21 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scal } /** -* @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* -* @param input pointer to the start of the input float values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @param blocks number of 512-bit blocks to compress. -* @return int status code (0 for success). -* -* @note This function requires AVX2 and BMI2 support. -*/ + * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * + * @param input pointer to the start of the input float values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @param blocks number of 512-bit blocks to compress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr, const u32 blocks) { +int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -282,7 +272,7 @@ int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 sca for (u32 block = 0; block < blocks; block++) { mm256_compress_block_bmi2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } @@ -290,21 +280,20 @@ int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 sca } /** -* @brief Compress multiple blocks of double values using AVX2 and BMI2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @param input pointer to the start of the input double values. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where compressed bytes will be stored. -* @param blocks number of blocks to compress. -* @return int status code (0 for success). -* -* @note This function requires AVX2 and BMI2 support. -*/ + * @brief Compress multiple blocks of double values using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @param input pointer to the start of the input double values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @param blocks number of blocks to compress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr, const u32 blocks) { +int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -313,12 +302,12 @@ int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 sca for (u32 block = 0; block < blocks; block++) { mm256_compress_block_bmi2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } return 0; } -} // namespace pernix +} // namespace pernix #endif // PERNIX_BMI2_COMPRESSION_H diff --git a/include/pernix/x86/bmi2/bmi2_decompression.h b/include/pernix/x86/bmi2/bmi2_decompression.h index 69bef94..cc64ec1 100644 --- a/include/pernix/x86/bmi2/bmi2_decompression.h +++ b/include/pernix/x86/bmi2/bmi2_decompression.h @@ -1,24 +1,23 @@ #ifndef PERNIX_BMI2_DECOMPRESSION_H #define PERNIX_BMI2_DECOMPRESSION_H -#include #include +#include #include #include #include #include - namespace pernix { namespace internal { /** -* @brief Sign-extend packed values after BMI2 expansion into 32-bit lanes. -* -* @tparam BIT_WIDTH original encoded bit width. -* @param source register containing unpacked values. -* @return __m128i sign-extended values. -*/ + * @brief Sign-extend packed values after BMI2 expansion into 32-bit lanes. + * + * @tparam BIT_WIDTH original encoded bit width. + * @param source register containing unpacked values. + * @return __m128i sign-extended values. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __m128i mm_sign_extend32(__m128i source) { @@ -33,12 +32,12 @@ __m128i mm_sign_extend32(__m128i source) { } /** -* @brief Sign-extend packed values after BMI2 expansion into eight 32-bit lanes. -* -* @tparam BIT_WIDTH original encoded bit width. -* @param source register containing unpacked values. -* @return __m256i sign-extended values. -*/ + * @brief Sign-extend packed values after BMI2 expansion into eight 32-bit lanes. + * + * @tparam BIT_WIDTH original encoded bit width. + * @param source register containing unpacked values. + * @return __m256i sign-extended values. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __m256i mm256_sign_extend32(__m256i source) { @@ -53,13 +52,13 @@ __m256i mm256_sign_extend32(__m256i source) { } /** -* @brief Unpack four values from a BMI2-packed input buffer. -* -* @tparam BIT_WIDTH bit width per packed value. -* @tparam SIGN_VALUES whether to sign-extend the unpacked values. -* @param input pointer to the packed input buffer. -* @return __m128i unpacked values. -*/ + * @brief Unpack four values from a BMI2-packed input buffer. + * + * @tparam BIT_WIDTH bit width per packed value. + * @tparam SIGN_VALUES whether to sign-extend the unpacked values. + * @param input pointer to the packed input buffer. + * @return __m128i unpacked values. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH > 0 && BIT_WIDTH <= 24) __m128i mm_unpack_epi32_bmi2(const u8* __restrict__ input) { @@ -112,13 +111,13 @@ __m128i mm_unpack_epi32_bmi2(const u8* __restrict__ input) { } /** -* @brief Unpack eight values from a BMI2-packed input buffer. -* -* @tparam BIT_WIDTH bit width per packed value. -* @tparam SIGN_VALUES whether to sign-extend the unpacked values. -* @param input pointer to the packed input buffer. -* @return __m256i unpacked values. -*/ + * @brief Unpack eight values from a BMI2-packed input buffer. + * + * @tparam BIT_WIDTH bit width per packed value. + * @tparam SIGN_VALUES whether to sign-extend the unpacked values. + * @param input pointer to the packed input buffer. + * @return __m256i unpacked values. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __m256i mm256_unpack_epi32_bmi2(const u8* __restrict__ input) { @@ -171,8 +170,7 @@ __m256i mm256_unpack_epi32_bmi2(const u8* __restrict__ input) { alignas(16) u64 raw_values[2]{}; std::memcpy(raw_values, input + second_group_byte_offset, packed_bytes - second_group_byte_offset); - temp_values[2] = (raw_values[0] >> second_group_shift) | ( - raw_values[1] << (64 - second_group_shift)); + temp_values[2] = (raw_values[0] >> second_group_shift) | (raw_values[1] << (64 - second_group_shift)); temp_values[3] = raw_values[1] >> second_group_shift; } @@ -182,8 +180,7 @@ __m256i mm256_unpack_epi32_bmi2(const u8* __restrict__ input) { values[2] = _pdep_u64((temp_values[2]), pdep_mask); values[3] = _pdep_u64((temp_values[2] >> shift1) | (temp_values[3] << shift2), pdep_mask); - result = _mm256_set_epi64x(static_cast(values[3]), static_cast(values[2]), - static_cast(values[1]), + result = _mm256_set_epi64x(static_cast(values[3]), static_cast(values[2]), static_cast(values[1]), static_cast(values[0])); } @@ -192,26 +189,25 @@ __m256i mm256_unpack_epi32_bmi2(const u8* __restrict__ input) { } return result; } -} // namespace internal +} // namespace internal /** -* @brief Decompress a single 512\-bit block using AVX2 and BMI2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). -* -* @param input pointer to the start of the compressed block. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed float values will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX2 and BMI2 support. -*/ + * @brief Decompress a single 512\-bit block using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). + * + * @param input pointer to the start of the compressed block. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed float values will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr) { +int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -225,14 +221,12 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f32 sc const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); _mm256_storeu_ps(output, dequantized); - input += BIT_WIDTH; + input += BIT_WIDTH; output += 8; } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback - (input, remaining); + const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); for (u32 i = 0; i < remaining; i++) { output[i] = internal::dequantize_epi32(tail_values[i], scale); } @@ -242,23 +236,22 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f32 sc } /** -* @brief Decompress a single block to double values using AVX2 and BMI2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). -* -* @param input pointer to the start of the compressed block. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed double values will be stored. -* @return int status code (0 for success). -* -* @note This function requires AVX2 and BMI2 support. -*/ + * @brief Decompress a single block to double values using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 16). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). + * + * @param input pointer to the start of the compressed block. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed double values will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr) { +int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -278,14 +271,12 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f64 sc _mm256_storeu_pd(output, dequantized1); _mm256_storeu_pd(output + 4, dequantized2); - input += BIT_WIDTH; + input += BIT_WIDTH; output += 8; } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback - (input, remaining); + const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); for (u32 i = 0; i < remaining; i++) { output[i] = internal::dequantize_epi64(tail_values[i], scale); } @@ -295,25 +286,23 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f64 sc } /** -* @brief Decompress multiple 512\-bit blocks using AVX2 and BMI2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). -* -* @param input pointer to the start of the compressed data. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed float values will be stored. -* @param blocks number of 512-bit blocks to decompress. -* @return int status code (0 for success). -* -* @note This function requires AVX2 and BMI2 support. -*/ + * @brief Decompress multiple 512\-bit blocks using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). + * + * @param input pointer to the start of the compressed data. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed float values will be stored. + * @param blocks number of 512-bit blocks to decompress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 scale, - void* __restrict__ output_ptr, - const u32 blocks) { +int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -322,7 +311,7 @@ int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 s for (u32 block = 0; block < blocks; block++) { mm256_decompress_block_bmi2(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -330,25 +319,23 @@ int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 s } /** -* @brief Decompress multiple blocks to double values using AVX2 and BMI2 instructions. -* -* @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). -* @tparam SIGN_VALUES whether the values are signed or unsigned. -* @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). -* -* @param input pointer to the start of the compressed data. -* @param scale scaling factor used during quantization. -* @param output pointer to the output buffer where decompressed double values will be stored. -* @param blocks number of blocks to decompress. -* @return int status code (0 for success). -* -* @note This function requires AVX2 and BMI2 support. -*/ + * @brief Decompress multiple blocks to double values using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @tparam SIGN_VALUES whether the values are signed or unsigned. + * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). + * + * @param input pointer to the start of the compressed data. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where decompressed double values will be stored. + * @param blocks number of blocks to decompress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 scale, - void* __restrict__ output_ptr, - const u32 blocks) { +int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { const auto* input = static_cast(input_ptr); auto* output = static_cast(output_ptr); @@ -357,12 +344,12 @@ int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 s for (u32 block = 0; block < blocks; block++) { mm256_decompress_block_bmi2(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix +} // namespace pernix #endif // PERNIX_BMI2_DECOMPRESSION_H diff --git a/include/pernix/x86/utils.h b/include/pernix/x86/utils.h index 2347eb6..3401b03 100644 --- a/include/pernix/x86/utils.h +++ b/include/pernix/x86/utils.h @@ -9,6 +9,6 @@ static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements const u32 tail_bytes = (tail_bits + 7u) / 8u; return tail_bytes; } -} // namespace pernix::x86::internal +} // namespace pernix::x86::internal #endif // PERNIX_X86_UTILS_H diff --git a/src/arm64/neon/compression.cpp b/src/arm64/neon/compression.cpp index 3594303..5278adb 100644 --- a/src/arm64/neon/compression.cpp +++ b/src/arm64/neon/compression.cpp @@ -1,28 +1,28 @@ -#include #include +#include namespace pernix::internal { - Kernel select_neon_compress_block_f32(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"neon", nullptr}; - } +Kernel select_neon_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} - Kernel select_neon_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"neon", nullptr}; - } +Kernel select_neon_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} - Kernel select_neon_compress_block_f64(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"neon", nullptr}; - } +Kernel select_neon_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} - Kernel select_neon_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"neon", nullptr}; - } +Kernel select_neon_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; } +} // namespace pernix::internal diff --git a/src/arm64/neon/decompression.cpp b/src/arm64/neon/decompression.cpp index 3083926..736498c 100644 --- a/src/arm64/neon/decompression.cpp +++ b/src/arm64/neon/decompression.cpp @@ -1,42 +1,42 @@ -#include #include +#include using pernix::arm64::neon::neon_decompress_block; using pernix::arm64::neon::neon_decompress_blocks; namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_block); \ - return Kernel("neon", &neon_decompress_block) +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ - return Kernel("neon", &neon_decompress_blocks) +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_block); \ - return Kernel("neon", &neon_decompress_block) +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ - return Kernel("neon", &neon_decompress_blocks) +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -52,53 +52,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"neon", nullptr}; \ - } \ + default: \ + return {"neon", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"neon", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -114,39 +116,41 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"neon", nullptr}; \ - } \ + default: \ + return {"neon", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"neon", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ break Kernel select_neon_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { @@ -205,4 +209,4 @@ Kernel select_neon_decompress_blocks_f64(const u8 bit_width #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal diff --git a/src/arm64/sve2/compression.cpp b/src/arm64/sve2/compression.cpp index 1839f12..262254d 100644 --- a/src/arm64/sve2/compression.cpp +++ b/src/arm64/sve2/compression.cpp @@ -1,28 +1,28 @@ -#include #include +#include namespace pernix::internal { - Kernel select_sve2_compress_block_f32(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"sve2", nullptr}; - } +Kernel select_sve2_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} - Kernel select_sve2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"sve2", nullptr}; - } +Kernel select_sve2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} - Kernel select_sve2_compress_block_f64(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"sve2", nullptr}; - } +Kernel select_sve2_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} - Kernel select_sve2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - (void) bit_width; - (void) block_size; - return {"sve2", nullptr}; - } +Kernel select_sve2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; } +} // namespace pernix::internal diff --git a/src/arm64/sve2/decompression.cpp b/src/arm64/sve2/decompression.cpp index 8e65f6e..16c8e35 100644 --- a/src/arm64/sve2/decompression.cpp +++ b/src/arm64/sve2/decompression.cpp @@ -1,42 +1,42 @@ -#include #include +#include using pernix::arm64::sve2::sve2_decompress_block; using pernix::arm64::sve2::sve2_decompress_blocks; namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ - return Kernel("sve2", &sve2_decompress_block) +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ - return Kernel("sve2", &sve2_decompress_blocks) +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ - return Kernel("sve2", &sve2_decompress_block) +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ - return Kernel("sve2", &sve2_decompress_blocks) +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -52,53 +52,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"sve2", nullptr}; \ - } \ + default: \ + return {"sve2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"sve2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -114,88 +116,90 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"sve2", nullptr}; \ - } \ + default: \ + return {"sve2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"sve2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ break - Kernel select_sve2_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"sve2", nullptr}; - } +Kernel select_sve2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"sve2", nullptr}; } +} - Kernel select_sve2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"sve2", nullptr}; - } +Kernel select_sve2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"sve2", nullptr}; } +} - Kernel select_sve2_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"sve2", nullptr}; - } +Kernel select_sve2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"sve2", nullptr}; } +} - Kernel select_sve2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"sve2", nullptr}; - } +Kernel select_sve2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"sve2", nullptr}; } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -205,4 +209,4 @@ case N: \ #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal diff --git a/src/dispatch/cpu_features_arm.cpp b/src/dispatch/cpu_features_arm.cpp index b31cf8b..516041e 100644 --- a/src/dispatch/cpu_features_arm.cpp +++ b/src/dispatch/cpu_features_arm.cpp @@ -25,8 +25,8 @@ CpuFeatures detect_cpu_features() { #if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) unsigned long hwcap = getauxval(AT_HWCAP); unsigned long hwcap2 = getauxval(AT_HWCAP2); - features.sve = (hwcap & HWCAP_SVE) != 0; - features.sve2 = (hwcap2 & HWCAP2_SVE2) != 0; + features.sve = (hwcap & HWCAP_SVE) != 0; + features.sve2 = (hwcap2 & HWCAP2_SVE2) != 0; #endif return features; @@ -36,4 +36,4 @@ CpuFeatures get_cached_cpu_features() { static const CpuFeatures features = detect_cpu_features(); return features; } -} +} // namespace pernix::internal diff --git a/src/dispatch/cpu_features_x86.cpp b/src/dispatch/cpu_features_x86.cpp index 9abe896..94496e2 100644 --- a/src/dispatch/cpu_features_x86.cpp +++ b/src/dispatch/cpu_features_x86.cpp @@ -1,6 +1,5 @@ -#include - #include +#include #if defined(__x86_64__) || defined(_M_X64) || defined(__i386) || defined(_M_IX86) #if defined(_MSC_VER) @@ -10,7 +9,6 @@ #include #endif - namespace pernix::internal { namespace { #if defined(_MSC_VER) @@ -41,7 +39,7 @@ u64 xgetbv(unsigned int index) { bool bit_set(int value, int bit) { return (value & (1 << bit)) != 0; } -} // namespace +} // namespace CpuFeatures detect_cpu_features() { CpuFeatures features{}; @@ -61,10 +59,7 @@ CpuFeatures detect_cpu_features() { const bool xmm_enabled = (xcr0 & 0x2) != 0; const bool ymm_enabled = (xcr0 & 0x4) != 0; - const bool zmm_enabled = - (xcr0 & 0x20) != 0 && - (xcr0 & 0x40) != 0 && - (xcr0 & 0x80) != 0; + const bool zmm_enabled = (xcr0 & 0x20) != 0 && (xcr0 & 0x40) != 0 && (xcr0 & 0x80) != 0; if (!xmm_enabled || !ymm_enabled) { return features; @@ -90,7 +85,7 @@ CpuFeatures get_cached_cpu_features() { static const CpuFeatures features = detect_cpu_features(); return features; } -} // namespace pernix::internal +} // namespace pernix::internal #else namespace pernix::internal { @@ -102,6 +97,6 @@ CpuFeatures get_cached_cpu_features() { static const CpuFeatures features = detect_cpu_features(); return features; } -} // namespace pernix::internal +} // namespace pernix::internal #endif diff --git a/src/dispatch/select.cpp b/src/dispatch/select.cpp index d4e2264..e2851db 100644 --- a/src/dispatch/select.cpp +++ b/src/dispatch/select.cpp @@ -1,913 +1,868 @@ -#include #include +#include namespace pernix::internal { - bool is_optional_fallback_backend(const Backend backend) { - switch (backend) { - case Backend::FallbackStdpar: - case Backend::FallbackSimd: - return true; - default: - return false; - } +bool is_optional_fallback_backend(const Backend backend) { + switch (backend) { + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + default: + return false; } +} - bool is_compiled_backend(const Backend backend) { - switch (backend) { - case Backend::Auto: - case Backend::Fallback: - return true; +bool is_compiled_backend(const Backend backend) { + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + return true; #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return true; + case Backend::FallbackStdpar: + return true; #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return true; + case Backend::FallbackSimd: + return true; #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return true; + case Backend::X86Avx2: + return true; #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return true; + case Backend::X86Bmi2: + return true; #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return true; + case Backend::X86Avx512Vbmi: + return true; #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return true; + case Backend::Arm64Neon: + return true; #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: - return true; -#endif - default: - return false; - } - } - - bool is_backend_supported_on_machine(const Backend backend) { - const CpuFeatures features = get_cached_cpu_features(); - - switch (backend) { - case Backend::Auto: - case Backend::Fallback: - case Backend::FallbackStdpar: - case Backend::FallbackSimd: - return true; - case Backend::X86Avx2: - return features.avx2; - case Backend::X86Bmi2: - return features.avx2 && features.bmi2; - case Backend::X86Avx512Vbmi: - return features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && - features.avx512vbmi; - case Backend::Arm64Neon: - return features.neon; - case Backend::Arm64Sve: - return features.sve2; - default: - return false; - } - } - - Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_compress_block_f32(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_block_f32(bit_width, block_size); + case Backend::Arm64Sve: + return true; +#endif + default: + return false; + } +} + +bool is_backend_supported_on_machine(const Backend backend) { + const CpuFeatures features = get_cached_cpu_features(); + + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + case Backend::X86Avx2: + return features.avx2; + case Backend::X86Bmi2: + return features.avx2 && features.bmi2; + case Backend::X86Avx512Vbmi: + return features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi; + case Backend::Arm64Neon: + return features.neon; + case Backend::Arm64Sve: + return features.sve2; + default: + return false; + } +} + +Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f32(bit_width, block_size); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_compress_block_f32(bit_width, block_size); + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_compress_block_f32(bit_width, block_size); + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_block_f32(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_block_f32(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_block_f32(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_block_f32(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_block_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_compress_block_f32(bit_width, block_size); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f32(bit_width, block_size); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_compress_blocks_f32(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_blocks_f32(bit_width, block_size); +Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f32(bit_width, block_size); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_compress_blocks_f32(bit_width, block_size); + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_compress_blocks_f32(bit_width, block_size); + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_blocks_f32(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_blocks_f32(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_blocks_f32(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_blocks_f32(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_compress_blocks_f32(bit_width, block_size); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f32(bit_width, block_size); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_compress_block_f64(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_block_f64(bit_width, block_size); +Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f64(bit_width, block_size); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_compress_block_f64(bit_width, block_size); + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_compress_block_f64(bit_width, block_size); + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_block_f64(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_block_f64(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_block_f64(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_block_f64(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_block_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_compress_block_f64(bit_width, block_size); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f64(bit_width, block_size); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_compress_blocks_f64(bit_width, block_size); - case Backend::Fallback: - return select_fallback_compress_blocks_f64(bit_width, block_size); +Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f64(bit_width, block_size); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_compress_blocks_f64(bit_width, block_size); + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_compress_blocks_f64(bit_width, block_size); + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_compress_blocks_f64(bit_width, block_size); + case Backend::X86Avx2: + return select_avx2_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_compress_blocks_f64(bit_width, block_size); + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_compress_blocks_f64(bit_width, block_size); + case Backend::Arm64Neon: + return select_neon_compress_blocks_f64(bit_width, block_size); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_compress_blocks_f64(bit_width, block_size); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f64(bit_width, block_size); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, - bool sign_values) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_decompress_block_f32(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_block_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, - bool sign_values) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, - bool sign_values) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_decompress_block_f64(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_block_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, - bool sign_values) { - if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { - return {"unsupported_implementation", nullptr}; - } - if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && - !is_backend_supported_on_machine(backend)) { - return {"unsupported_implementation", nullptr}; - } - switch (backend) { - case Backend::Auto: - return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); - case Backend::Fallback: - return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - case Backend::FallbackStdpar: - return select_fallback_stdpar_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_FALLBACK_SIMD) - case Backend::FallbackSimd: - return select_fallback_simd_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - case Backend::X86Avx512Vbmi: - return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_AVX2) - case Backend::X86Avx2: - return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_X86_BMI2) - case Backend::X86Bmi2: - return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_NEON) - case Backend::Arm64Neon: - return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - case Backend::Arm64Sve: { - if (get_cached_cpu_features().sve2) { - return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); - } - return {"invalid_backend", nullptr}; + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); } -#endif - default: - return {"invalid_backend", nullptr}; + return {"invalid_backend", nullptr}; } +#endif + default: + return {"invalid_backend", nullptr}; } +} - Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { + return kernel; } + } #endif - return select_fallback_compress_block_f32(bit_width, block_size); - } + return select_fallback_compress_block_f32(bit_width, block_size); +} - Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { + return kernel; } + } #endif - return select_fallback_compress_blocks_f32(bit_width, block_size); - } + return select_fallback_compress_blocks_f32(bit_width, block_size); +} - Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { + return kernel; } + } #endif - return select_fallback_compress_block_f64(bit_width, block_size); - } + return select_fallback_compress_block_f64(bit_width, block_size); +} - Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { + return kernel; } + } #endif - return select_fallback_compress_blocks_f64(bit_width, block_size); - } + return select_fallback_compress_blocks_f64(bit_width, block_size); +} - Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif - return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); - } + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +} - Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; } + } #endif - return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); - } + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +} - Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif - return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); - } + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +} - Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values) { -#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) - const CpuFeatures features = get_cached_cpu_features(); +Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); #endif #if defined(PERNIX_BUILD_X86_AVX512_VBMI) - if ( - features.avx512f && - features.avx512dq && - features.avx512bw && - features.avx512vl && - features.avx512vbmi - ) { - if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_AVX2) - if (features.avx2) { - if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_X86_BMI2) - if (features.bmi2) { - if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_NEON) - if (features.neon) { - if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif #if defined(PERNIX_BUILD_ARM64_SVE2) - if (features.sve2) { - if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { - return kernel; - } + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; } + } #endif - return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); - } + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); } +} // namespace pernix::internal diff --git a/src/pernix.cpp b/src/pernix.cpp index 6adbd95..0479c25 100644 --- a/src/pernix.cpp +++ b/src/pernix.cpp @@ -25,87 +25,83 @@ u32 pernix_elements_per_block(u8 bit_width) { return pernix::detail::elements_per_block(bit_width); } -const char *pernix_status_string(pernix_status status) { +const char* pernix_status_string(pernix_status status) { return pernix::detail::status_string(status); } -pernix_status pernix_scale_f32(float bmax, u8 bit_width, float *scale) { +pernix_status pernix_scale_f32(float bmax, u8 bit_width, float* scale) { return pernix::detail::scale_from_bmax(bmax, bit_width, scale); } -pernix_status pernix_scale_f64(double bmax, u8 bit_width, double *scale) { +pernix_status pernix_scale_f64(double bmax, u8 bit_width, double* scale) { return pernix::detail::scale_from_bmax(bmax, bit_width, scale); } -pernix_status pernix_decompression_scale_f32(float bmax, u8 bit_width, float *scale) { +pernix_status pernix_decompression_scale_f32(float bmax, u8 bit_width, float* scale) { return pernix_scale_f32(bmax, bit_width, scale); } -pernix_status pernix_decompression_scale_f64(double bmax, u8 bit_width, double *scale) { +pernix_status pernix_decompression_scale_f64(double bmax, u8 bit_width, double* scale) { return pernix_scale_f64(bmax, bit_width, scale); } -pernix_status pernix_inverse_scale_f32(float scale, float *inverse_scale) { +pernix_status pernix_inverse_scale_f32(float scale, float* inverse_scale) { return pernix::detail::inverse_scale(scale, inverse_scale); } -pernix_status pernix_inverse_scale_f64(double scale, double *inverse_scale) { +pernix_status pernix_inverse_scale_f64(double scale, double* inverse_scale) { return pernix::detail::inverse_scale(scale, inverse_scale); } -pernix_status pernix_compression_scale_f32(float bmax, u8 bit_width, float *inverse_scale) { +pernix_status pernix_compression_scale_f32(float bmax, u8 bit_width, float* inverse_scale) { return pernix::detail::compression_scale_from_bmax(bmax, bit_width, inverse_scale); } -pernix_status pernix_compression_scale_f64(double bmax, u8 bit_width, double *inverse_scale) { +pernix_status pernix_compression_scale_f64(double bmax, u8 bit_width, double* inverse_scale) { return pernix::detail::compression_scale_from_bmax(bmax, bit_width, inverse_scale); } -pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, - float scale, void *output) { - return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, - output); +pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output) { + return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, output); } -pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, - float scale, void *output, u32 blocks) { - return pernix::detail::compress_blocks(static_cast(backend), bit_width, block_size, input, - scale, output, blocks); +pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, void* output, + u32 blocks) { + return pernix::detail::compress_blocks(static_cast(backend), bit_width, block_size, input, scale, output, blocks); } -pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, - float scale, void *output, bool sign_values) { - return pernix::detail::decompress_block(static_cast(backend), bit_width, block_size, input, - scale, output, sign_values); +pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output, bool sign_values) { + return pernix::detail::decompress_block(static_cast(backend), bit_width, block_size, input, scale, output, + sign_values); } -pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, - float scale, void *output, u32 blocks, bool sign_values) { - return pernix::detail::decompress_blocks(static_cast(backend), bit_width, block_size, input, - scale, output, blocks, sign_values); +pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output, u32 blocks, bool sign_values) { + return pernix::detail::decompress_blocks(static_cast(backend), bit_width, block_size, input, scale, output, blocks, + sign_values); } -pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, - double scale, void *output) { - return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, - output); +pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output) { + return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, output); } -pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, - double scale, void *output, u32 blocks) { - return pernix::detail::compress_blocks(static_cast(backend), bit_width, block_size, input, - scale, output, blocks); +pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, u32 blocks) { + return pernix::detail::compress_blocks(static_cast(backend), bit_width, block_size, input, scale, output, blocks); } -pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, - double scale, void *output, bool sign_values) { - return pernix::detail::decompress_block(static_cast(backend), bit_width, block_size, input, - scale, output, sign_values); +pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, bool sign_values) { + return pernix::detail::decompress_block(static_cast(backend), bit_width, block_size, input, scale, output, + sign_values); } -pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void *input, - double scale, void *output, u32 blocks, bool sign_values) { - return pernix::detail::decompress_blocks(static_cast(backend), bit_width, block_size, input, - scale, output, blocks, sign_values); +pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, u32 blocks, bool sign_values) { + return pernix::detail::decompress_blocks(static_cast(backend), bit_width, block_size, input, scale, output, blocks, + sign_values); } } diff --git a/src/x86/avx2/avx2_compression.cpp b/src/x86/avx2/avx2_compression.cpp index 548989b..0ce0189 100644 --- a/src/x86/avx2/avx2_compression.cpp +++ b/src/x86/avx2/avx2_compression.cpp @@ -3,29 +3,33 @@ namespace pernix::internal { #define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ -case N: return Kernel("avx2", &mm256_compress_block_avx2) + case N: \ + return Kernel("avx2", &mm256_compress_block_avx2) #define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ -case N: return Kernel("avx2", &mm256_compress_blocks_avx2) + case N: \ + return Kernel("avx2", &mm256_compress_blocks_avx2) #define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ -case N: return Kernel("avx2", &mm256_compress_block_avx2) + case N: \ + return Kernel("avx2", &mm256_compress_block_avx2) #define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ -case N: return Kernel("avx2", &mm256_compress_blocks_avx2) + case N: \ + return Kernel("avx2", &mm256_compress_blocks_avx2) -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ @@ -41,51 +45,53 @@ case N: return Kernel("avx2", &mm256_compress_blocks_avx2("avx2", &mm256_compress_blocks_avx2 select_avx2_compress_block_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"avx2", nullptr}; - } +Kernel select_avx2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; } +} - Kernel select_avx2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"avx2", nullptr}; - } +Kernel select_avx2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; } +} - Kernel select_avx2_compress_block_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"avx2", nullptr}; - } +Kernel select_avx2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; } +} - Kernel select_avx2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"avx2", nullptr}; - } +Kernel select_avx2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; } +} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 @@ -190,4 +198,4 @@ case N: return Kernel("avx2", &mm256_compress_blocks_avx2 namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ - return Kernel("avx2", &mm256_decompress_block_avx2) +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ - return Kernel("avx2", &mm256_decompress_blocks_avx2) +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ - return Kernel("avx2", &mm256_decompress_block_avx2) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ - return Kernel("avx2", &mm256_decompress_blocks_avx2) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -49,53 +48,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"avx2", nullptr}; \ - } \ + default: \ + return {"avx2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"avx2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -111,88 +112,90 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"avx2", nullptr}; \ - } \ + default: \ + return {"avx2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"avx2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ break - Kernel select_avx2_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"avx2", nullptr}; - } +Kernel select_avx2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; } +} - Kernel select_avx2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"avx2", nullptr}; - } +Kernel select_avx2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"avx2", nullptr}; } +} - Kernel select_avx2_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"avx2", nullptr}; - } +Kernel select_avx2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; } +} - Kernel select_avx2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"avx2", nullptr}; - } +Kernel select_avx2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"avx2", nullptr}; } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -202,4 +205,4 @@ case N: \ #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal diff --git a/src/x86/avx512vbmi/avx512vbmi_compression.cpp b/src/x86/avx512vbmi/avx512vbmi_compression.cpp index 8eb25c5..03e613f 100644 --- a/src/x86/avx512vbmi/avx512vbmi_compression.cpp +++ b/src/x86/avx512vbmi/avx512vbmi_compression.cpp @@ -3,29 +3,33 @@ namespace pernix::internal { #define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ -case N: return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + case N: \ + return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) #define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ -case N: return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + case N: \ + return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) #define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ -case N: return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + case N: \ + return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) #define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ -case N: return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + case N: \ + return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ @@ -41,51 +45,53 @@ case N: return Kernel("avx512vbmi", &mm512_compress_blocks_ PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ + default: \ + return {"avx512vbmi", nullptr}; \ } #define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ } -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ @@ -101,86 +107,88 @@ case N: return Kernel("avx512vbmi", &mm512_compress_blocks_ PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ + default: \ + return {"avx512vbmi", nullptr}; \ } #define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ } - Kernel select_avx512vbmi_compress_block_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"avx512vbmi", nullptr}; - } +Kernel select_avx512vbmi_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; } +} - Kernel select_avx512vbmi_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"avx512vbmi", nullptr}; - } +Kernel select_avx512vbmi_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; } +} - Kernel select_avx512vbmi_compress_block_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"avx512vbmi", nullptr}; - } +Kernel select_avx512vbmi_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; } +} - Kernel select_avx512vbmi_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"avx512vbmi", nullptr}; - } +Kernel select_avx512vbmi_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; } +} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 @@ -190,4 +198,4 @@ case N: return Kernel("avx512vbmi", &mm512_compress_blocks_ #undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 #undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 -} +} // namespace pernix::internal diff --git a/src/x86/avx512vbmi/avx512vbmi_decompression.cpp b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp index ac2cbcd..4ebb3b7 100644 --- a/src/x86/avx512vbmi/avx512vbmi_decompression.cpp +++ b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp @@ -2,38 +2,37 @@ #include namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ - return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ - return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ - return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ - return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -49,53 +48,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -111,88 +112,90 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"avx512vbmi", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ break - Kernel select_avx512vbmi_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"avx512vbmi", nullptr}; - } +Kernel select_avx512vbmi_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; } +} - Kernel select_avx512vbmi_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"avx512vbmi", nullptr}; - } +Kernel select_avx512vbmi_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"avx512vbmi", nullptr}; } +} - Kernel select_avx512vbmi_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"avx512vbmi", nullptr}; - } +Kernel select_avx512vbmi_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; } +} - Kernel select_avx512vbmi_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"avx512vbmi", nullptr}; - } +Kernel select_avx512vbmi_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"avx512vbmi", nullptr}; } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -202,4 +205,4 @@ case N: \ #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal diff --git a/src/x86/bmi2/bmi2_compression.cpp b/src/x86/bmi2/bmi2_compression.cpp index e978ba2..9920ced 100644 --- a/src/x86/bmi2/bmi2_compression.cpp +++ b/src/x86/bmi2/bmi2_compression.cpp @@ -3,29 +3,33 @@ namespace pernix::internal { #define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ -case N: return Kernel("bmi2", &mm256_compress_block_bmi2) + case N: \ + return Kernel("bmi2", &mm256_compress_block_bmi2) #define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ -case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2) + case N: \ + return Kernel("bmi2", &mm256_compress_blocks_bmi2) #define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ -case N: return Kernel("bmi2", &mm256_compress_block_bmi2) + case N: \ + return Kernel("bmi2", &mm256_compress_block_bmi2) #define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ -case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2) + case N: \ + return Kernel("bmi2", &mm256_compress_blocks_bmi2) -#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ @@ -41,51 +45,53 @@ case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2("bmi2", &mm256_compress_blocks_bmi2 select_bmi2_compress_block_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); - default: - return {"bmi2", nullptr}; - } +Kernel select_bmi2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; } +} - Kernel select_bmi2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); - default: - return {"bmi2", nullptr}; - } +Kernel select_bmi2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; } +} - Kernel select_bmi2_compress_block_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); - default: - return {"bmi2", nullptr}; - } +Kernel select_bmi2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; } +} - Kernel select_bmi2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { - switch (block_size) { - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); - default: - return {"bmi2", nullptr}; - } +Kernel select_bmi2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; } +} #undef PERNIX_CASE_COMPRESS_BLOCK_32 #undef PERNIX_CASE_COMPRESS_BLOCKS_32 @@ -190,4 +198,4 @@ case N: return Kernel("bmi2", &mm256_compress_blocks_bmi2 namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ - return Kernel("bmi2", &mm256_decompress_block_bmi2) +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ -case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ - return Kernel("bmi2", &mm256_decompress_blocks_bmi2) +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ - return Kernel("bmi2", &mm256_decompress_block_bmi2) - -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ -case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ - return Kernel("bmi2", &mm256_decompress_blocks_bmi2) -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ @@ -49,53 +48,55 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ - default: return {"bmi2", nullptr}; \ - } \ + default: \ + return {"bmi2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ - default: return {"bmi2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ break -#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ @@ -111,88 +112,90 @@ case N: \ PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ - default: return {"bmi2", nullptr}; \ - } \ + default: \ + return {"bmi2", nullptr}; \ + } \ break #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ - case BS: \ - switch (bit_width) { \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ - PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ - default: return {"bmi2", nullptr}; \ - } \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ break - Kernel select_bmi2_decompress_block_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); - default: return {"bmi2", nullptr}; - } +Kernel select_bmi2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; } +} - Kernel select_bmi2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); - default: return {"bmi2", nullptr}; - } +Kernel select_bmi2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"bmi2", nullptr}; } +} - Kernel select_bmi2_decompress_block_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); - default: return {"bmi2", nullptr}; - } +Kernel select_bmi2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; } +} - Kernel select_bmi2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - bool sign_values) { - switch (block_size) { - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); - PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); - default: return {"bmi2", nullptr}; - } +Kernel select_bmi2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"bmi2", nullptr}; } +} #undef PERNIX_CASE_DECOMPRESS_BLOCK_32 #undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 @@ -202,4 +205,4 @@ case N: \ #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 #undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 -} +} // namespace pernix::internal diff --git a/tests/c_api_roundtrip.c b/tests/c_api_roundtrip.c index 49ce232..433c31e 100644 --- a/tests/c_api_roundtrip.c +++ b/tests/c_api_roundtrip.c @@ -19,9 +19,9 @@ int main(void) { float bmax = 0.0f; for (int i = 0; i < elements; ++i) { - input[i] = ((float)i - 16.0f) * 0.125f; + input[i] = ((float)i - 16.0f) * 0.125f; const float magnitude = input[i] < 0.0f ? -input[i] : input[i]; - bmax = bmax < magnitude ? magnitude : bmax; + bmax = bmax < magnitude ? magnitude : bmax; } float scale = 0.0f; @@ -29,20 +29,18 @@ int main(void) { return 1; } - pernix_status status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, - 1.0f / scale, compressed); + pernix_status status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 1.0f / scale, compressed); if (status != PERNIX_STATUS_OK) { return 1; } - status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, - true); + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, true); if (status != PERNIX_STATUS_OK) { return 2; } for (int i = 0; i < elements; ++i) { - const float diff = restored[i] - input[i]; + const float diff = restored[i] - input[i]; const float error = diff < 0.0f ? -diff : diff; if (error > scale) { return 3; diff --git a/tests/c_api_validation.c b/tests/c_api_validation.c index bc5f313..0e07be7 100644 --- a/tests/c_api_validation.c +++ b/tests/c_api_validation.c @@ -1,26 +1,21 @@ -#include - #include +#include #include -enum { - bit_width = 8, - block_size = 64, - elements = (block_size * 8) / bit_width -}; +enum { bit_width = 8, block_size = 64, elements = (block_size * 8) / bit_width }; static int expect_status(pernix_status actual, pernix_status expected) { return actual == expected ? 0 : 1; } int main(void) { - float input_f32[elements] = {0}; - float output_f32[elements] = {0}; - double input_f64[elements] = {0}; + float input_f32[elements] = {0}; + float output_f32[elements] = {0}; + double input_f64[elements] = {0}; double output_f64[elements] = {0}; - u8 compressed[block_size] = {0}; - float scale_f32 = 0.0f; - double scale_f64 = 0.0; + u8 compressed[block_size] = {0}; + float scale_f32 = 0.0f; + double scale_f64 = 0.0; if (expect_status(pernix_scale_f32(1.0f, bit_width, &scale_f32), PERNIX_STATUS_OK) != 0) { return 1; @@ -38,10 +33,8 @@ int main(void) { return 5; } - if (!pernix_is_valid_bit_width(pernix_min_bit_width()) || - !pernix_is_valid_bit_width(pernix_max_bit_width()) || - pernix_is_valid_bit_width(0) || - pernix_is_valid_bit_width((u8)(pernix_max_bit_width() + 1U))) { + if (!pernix_is_valid_bit_width(pernix_min_bit_width()) || !pernix_is_valid_bit_width(pernix_max_bit_width()) || + pernix_is_valid_bit_width(0) || pernix_is_valid_bit_width((u8)(pernix_max_bit_width() + 1U))) { return 6; } if (pernix_compressed_block_size() != block_size || pernix_elements_per_block(bit_width) != elements) { @@ -52,21 +45,17 @@ int main(void) { } if (strcmp(pernix_status_string(PERNIX_STATUS_OK), "PERNIX_STATUS_OK") != 0 || - strcmp(pernix_status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION), - "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION") != 0 || + strcmp(pernix_status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION), "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION") != 0 || strcmp(pernix_status_string((pernix_status)123), "PERNIX_STATUS_UNKNOWN") != 0) { return 23; } float decompression_scale_f32 = 0.0f; - float compression_scale_f32 = 0.0f; - float inverse_scale_f32 = 0.0f; - if (expect_status(pernix_decompression_scale_f32(127.0f, bit_width, &decompression_scale_f32), - PERNIX_STATUS_OK) != 0 || - expect_status(pernix_compression_scale_f32(127.0f, bit_width, &compression_scale_f32), - PERNIX_STATUS_OK) != 0 || - expect_status(pernix_inverse_scale_f32(decompression_scale_f32, &inverse_scale_f32), - PERNIX_STATUS_OK) != 0) { + float compression_scale_f32 = 0.0f; + float inverse_scale_f32 = 0.0f; + if (expect_status(pernix_decompression_scale_f32(127.0f, bit_width, &decompression_scale_f32), PERNIX_STATUS_OK) != 0 || + expect_status(pernix_compression_scale_f32(127.0f, bit_width, &compression_scale_f32), PERNIX_STATUS_OK) != 0 || + expect_status(pernix_inverse_scale_f32(decompression_scale_f32, &inverse_scale_f32), PERNIX_STATUS_OK) != 0) { return 24; } if (fabsf(decompression_scale_f32 - 1.0f) > 0.0f || fabsf(compression_scale_f32 - 1.0f) > 0.0f || @@ -74,25 +63,20 @@ int main(void) { return 25; } if (expect_status(pernix_inverse_scale_f32(0.0f, &inverse_scale_f32), PERNIX_STATUS_INVALID_ARGUMENT) != 0 || - expect_status(pernix_compression_scale_f32(1.0f, 0, &compression_scale_f32), - PERNIX_STATUS_INVALID_ARGUMENT) != 0 || + expect_status(pernix_compression_scale_f32(1.0f, 0, &compression_scale_f32), PERNIX_STATUS_INVALID_ARGUMENT) != 0 || expect_status(pernix_decompression_scale_f32(1.0f, bit_width, 0), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { return 26; } double decompression_scale_f64 = 0.0; - double compression_scale_f64 = 0.0; - double inverse_scale_f64 = 0.0; - if (expect_status(pernix_decompression_scale_f64(127.0, bit_width, &decompression_scale_f64), - PERNIX_STATUS_OK) != 0 || - expect_status(pernix_compression_scale_f64(127.0, bit_width, &compression_scale_f64), - PERNIX_STATUS_OK) != 0 || - expect_status(pernix_inverse_scale_f64(decompression_scale_f64, &inverse_scale_f64), - PERNIX_STATUS_OK) != 0) { + double compression_scale_f64 = 0.0; + double inverse_scale_f64 = 0.0; + if (expect_status(pernix_decompression_scale_f64(127.0, bit_width, &decompression_scale_f64), PERNIX_STATUS_OK) != 0 || + expect_status(pernix_compression_scale_f64(127.0, bit_width, &compression_scale_f64), PERNIX_STATUS_OK) != 0 || + expect_status(pernix_inverse_scale_f64(decompression_scale_f64, &inverse_scale_f64), PERNIX_STATUS_OK) != 0) { return 27; } - if (fabs(decompression_scale_f64 - 1.0) > 0.0 || fabs(compression_scale_f64 - 1.0) > 0.0 || - fabs(inverse_scale_f64 - 1.0) > 0.0) { + if (fabs(decompression_scale_f64 - 1.0) > 0.0 || fabs(compression_scale_f64 - 1.0) > 0.0 || fabs(inverse_scale_f64 - 1.0) > 0.0) { return 28; } @@ -104,8 +88,7 @@ int main(void) { PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE) != 0) { return 10; } - if (expect_status(pernix_compress_block_f32((pernix_backend)255, bit_width, block_size, input_f32, 1.0f, - compressed), + if (expect_status(pernix_compress_block_f32((pernix_backend)255, bit_width, block_size, input_f32, 1.0f, compressed), PERNIX_STATUS_UNSUPPORTED_BACKEND) != 0) { return 11; } @@ -117,49 +100,40 @@ int main(void) { PERNIX_STATUS_INVALID_ARGUMENT) != 0) { return 13; } - if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 0.0f, - compressed), + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 0.0f, compressed), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { return 14; } - if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0f, - output_f32, true), + if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0f, output_f32, true), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { return 15; } - if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0f, 0, - true), + if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0f, 0, true), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { return 16; } - if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, -1.0f, - output_f32, true), + if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, -1.0f, output_f32, true), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { return 17; } - if (expect_status(pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 1.0f, - compressed, 0), + if (expect_status(pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 1.0f, compressed, 0), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { return 18; } - if (expect_status(pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f64, 1.0, - compressed), + if (expect_status(pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f64, 1.0, compressed), PERNIX_STATUS_OK) != 0) { return 19; } - if (expect_status(pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0, - output_f64, true), + if (expect_status(pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0, output_f64, true), PERNIX_STATUS_OK) != 0) { return 20; } - if (expect_status(pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0, compressed, - 1), + if (expect_status(pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0, compressed, 1), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { return 21; } - if (expect_status(pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0, 0, - 1, true), + if (expect_status(pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0, 0, 1, true), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { return 22; } diff --git a/tests/deterministic_correctness_tests.cpp b/tests/deterministic_correctness_tests.cpp index 7265eca..692babe 100644 --- a/tests/deterministic_correctness_tests.cpp +++ b/tests/deterministic_correctness_tests.cpp @@ -1,11 +1,10 @@ #include -#include - #include #include #include #include +#include #include #include #include @@ -15,18 +14,9 @@ #if PERNIX_TEST_BLOCK_SIZE == 64 namespace { constexpr u32 kBlockSize = 64; -constexpr u32 kBlocks = 3; - -enum class Pattern { - Zero, - Constant, - Increasing, - Decreasing, - SmallMagnitude, - LargeMagnitude, - QuantizationBoundary, - PseudoRandom -}; +constexpr u32 kBlocks = 3; + +enum class Pattern { Zero, Constant, Increasing, Decreasing, SmallMagnitude, LargeMagnitude, QuantizationBoundary, PseudoRandom }; struct PatternCase { Pattern pattern; @@ -77,23 +67,22 @@ pernix_status compute_scale(const FloatT bmax, const u8 bit_width, FloatT& scale template FloatT tolerance_for_scale(const FloatT scale) { - return (abs_value(scale) * static_cast(0.55)) + - (std::numeric_limits::epsilon() * static_cast(64)); + return (abs_value(scale) * static_cast(0.55)) + (std::numeric_limits::epsilon() * static_cast(64)); } template std::vector make_dataset(const Pattern pattern, const u8 bit_width, const u32 blocks = kBlocks) { const u32 elements_per_block = pernix_elements_per_block(bit_width); std::vector values(static_cast(elements_per_block) * blocks); - const bool one_bit = bit_width == 1; + const bool one_bit = bit_width == 1; const FloatT amplitude = [&] { switch (pattern) { - case Pattern::SmallMagnitude: - return static_cast(1.0e-5); - case Pattern::LargeMagnitude: - return static_cast(1.0e6); - default: - return static_cast(16); + case Pattern::SmallMagnitude: + return static_cast(1.0e-5); + case Pattern::LargeMagnitude: + return static_cast(1.0e6); + default: + return static_cast(16); } }(); @@ -102,42 +91,40 @@ std::vector make_dataset(const Pattern pattern, const u8 bit_width, cons for (usize i = 0; i < values.size(); ++i) { const FloatT unit = values.size() > 1 - ? static_cast((static_cast(i) / static_cast(values.size() - 1)) * - 2.0 - 1.0) + ? static_cast((static_cast(i) / static_cast(values.size() - 1)) * 2.0 - 1.0) : FloatT{0}; switch (pattern) { - case Pattern::Zero: - values[i] = FloatT{0}; - break; - case Pattern::Constant: - values[i] = one_bit ? amplitude : amplitude / FloatT{2}; - break; - case Pattern::Increasing: - values[i] = one_bit ? static_cast(i % 2U) * amplitude : unit * amplitude; - break; - case Pattern::Decreasing: - values[i] = one_bit ? static_cast((i + 1U) % 2U) * amplitude : -unit * amplitude; - break; - case Pattern::SmallMagnitude: - case Pattern::LargeMagnitude: - values[i] = one_bit ? static_cast(i % 2U) * amplitude - : static_cast(((static_cast(i % 9U) - 4) / 4.0)) * amplitude; - break; - case Pattern::QuantizationBoundary: { - const FloatT step = amplitude / static_cast(17); - if (one_bit) { - values[i] = static_cast(i % 2U) * step; - } else { - const int bucket = static_cast(i % 9U) - 4; - const FloatT nudge = (i % 2U == 0U) ? step * static_cast(0.49) - : step * static_cast(-0.49); - values[i] = (static_cast(bucket) * step) + nudge; + case Pattern::Zero: + values[i] = FloatT{0}; + break; + case Pattern::Constant: + values[i] = one_bit ? amplitude : amplitude / FloatT{2}; + break; + case Pattern::Increasing: + values[i] = one_bit ? static_cast(i % 2U) * amplitude : unit * amplitude; + break; + case Pattern::Decreasing: + values[i] = one_bit ? static_cast((i + 1U) % 2U) * amplitude : -unit * amplitude; + break; + case Pattern::SmallMagnitude: + case Pattern::LargeMagnitude: + values[i] = one_bit ? static_cast(i % 2U) * amplitude + : static_cast(((static_cast(i % 9U) - 4) / 4.0)) * amplitude; + break; + case Pattern::QuantizationBoundary: { + const FloatT step = amplitude / static_cast(17); + if (one_bit) { + values[i] = static_cast(i % 2U) * step; + } else { + const int bucket = static_cast(i % 9U) - 4; + const FloatT nudge = (i % 2U == 0U) ? step * static_cast(0.49) : step * static_cast(-0.49); + values[i] = (static_cast(bucket) * step) + nudge; + } + break; } - break; - } - case Pattern::PseudoRandom: - values[i] = static_cast(uniform(rng)) * amplitude; - break; + case Pattern::PseudoRandom: + values[i] = static_cast(uniform(rng)) * amplitude; + break; } } @@ -154,99 +141,82 @@ void expect_near_input(const std::vector& input, const std::vector -pernix_status compress_blocks(pernix_backend backend, const u8 bit_width, const std::vector& input, - const FloatT inverse_scale, std::vector& compressed, const u32 blocks) { +pernix_status compress_blocks(pernix_backend backend, const u8 bit_width, const std::vector& input, const FloatT inverse_scale, + std::vector& compressed, const u32 blocks) { if constexpr (std::is_same_v) { - return pernix_compress_blocks_f32(backend, bit_width, kBlockSize, input.data(), inverse_scale, - compressed.data(), blocks); + return pernix_compress_blocks_f32(backend, bit_width, kBlockSize, input.data(), inverse_scale, compressed.data(), blocks); } else { - return pernix_compress_blocks_f64(backend, bit_width, kBlockSize, input.data(), inverse_scale, - compressed.data(), blocks); + return pernix_compress_blocks_f64(backend, bit_width, kBlockSize, input.data(), inverse_scale, compressed.data(), blocks); } } template -pernix_status decompress_blocks(pernix_backend backend, const u8 bit_width, const std::vector& compressed, - const FloatT scale, std::vector& output, const u32 blocks, - const bool sign_values = true) { +pernix_status decompress_blocks(pernix_backend backend, const u8 bit_width, const std::vector& compressed, const FloatT scale, + std::vector& output, const u32 blocks, const bool sign_values = true) { if constexpr (std::is_same_v) { - return pernix_decompress_blocks_f32(backend, bit_width, kBlockSize, compressed.data(), scale, output.data(), - blocks, sign_values); + return pernix_decompress_blocks_f32(backend, bit_width, kBlockSize, compressed.data(), scale, output.data(), blocks, sign_values); } else { - return pernix_decompress_blocks_f64(backend, bit_width, kBlockSize, compressed.data(), scale, output.data(), - blocks, sign_values); + return pernix_decompress_blocks_f64(backend, bit_width, kBlockSize, compressed.data(), scale, output.data(), blocks, sign_values); } } template void expect_fallback_round_trip(const u8 bit_width, const PatternCase& pattern_case) { - SCOPED_TRACE(::testing::Message() << "bit_width=" << static_cast(bit_width) - << ", pattern=" << pattern_case.name); + SCOPED_TRACE(::testing::Message() << "bit_width=" << static_cast(bit_width) << ", pattern=" << pattern_case.name); const auto input = make_dataset(pattern_case.pattern, bit_width); - FloatT scale = FloatT{0}; + FloatT scale = FloatT{0}; ASSERT_EQ(compute_scale(max_abs(input), bit_width, scale), PERNIX_STATUS_OK); std::vector compressed(kBlocks * kBlockSize); std::vector restored(input.size(), FloatT{}); - ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input, FloatT{1} / scale, compressed, kBlocks), - PERNIX_STATUS_OK); - ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed, scale, restored, kBlocks), - PERNIX_STATUS_OK); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input, FloatT{1} / scale, compressed, kBlocks), PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed, scale, restored, kBlocks), PERNIX_STATUS_OK); expect_near_input(input, restored, scale); } template void expect_backend_matches_fallback(const pernix_backend backend, const u8 bit_width) { - SCOPED_TRACE(::testing::Message() << "backend=" << static_cast(backend) - << ", bit_width=" << static_cast(bit_width)); + SCOPED_TRACE(::testing::Message() << "backend=" << static_cast(backend) << ", bit_width=" << static_cast(bit_width)); const auto input = make_dataset(Pattern::PseudoRandom, bit_width, 2); - FloatT scale = FloatT{0}; + FloatT scale = FloatT{0}; ASSERT_EQ(compute_scale(max_abs(input), bit_width, scale), PERNIX_STATUS_OK); std::vector fallback_compressed(2 * kBlockSize); std::vector backend_compressed(2 * kBlockSize); - ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input, FloatT{1} / scale, fallback_compressed, 2), - PERNIX_STATUS_OK); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input, FloatT{1} / scale, fallback_compressed, 2), PERNIX_STATUS_OK); const auto compress_status = compress_blocks(backend, bit_width, input, FloatT{1} / scale, backend_compressed, 2); if (compress_status != PERNIX_STATUS_OK) { - EXPECT_TRUE(compress_status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || - compress_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || + EXPECT_TRUE(compress_status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || compress_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || compress_status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); return; } std::vector fallback_restored(input.size(), FloatT{}); std::vector backend_restored(input.size(), FloatT{}); - ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, fallback_compressed, scale, fallback_restored, 2), - PERNIX_STATUS_OK); - ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, backend_compressed, scale, backend_restored, 2), - PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, fallback_compressed, scale, fallback_restored, 2), PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, backend_compressed, scale, backend_restored, 2), PERNIX_STATUS_OK); - const FloatT tolerance = (abs_value(scale) * static_cast(1.1)) + - (std::numeric_limits::epsilon() * static_cast(64)); + const FloatT tolerance = + (abs_value(scale) * static_cast(1.1)) + (std::numeric_limits::epsilon() * static_cast(64)); for (usize i = 0; i < input.size(); ++i) { - ASSERT_NEAR(backend_restored[i], fallback_restored[i], tolerance) - << "element=" << i << ", scale=" << scale; + ASSERT_NEAR(backend_restored[i], fallback_restored[i], tolerance) << "element=" << i << ", scale=" << scale; } std::vector backend_decompressed(input.size(), FloatT{}); - const auto decompress_status = - decompress_blocks(backend, bit_width, fallback_compressed, scale, backend_decompressed, 2); + const auto decompress_status = decompress_blocks(backend, bit_width, fallback_compressed, scale, backend_decompressed, 2); if (decompress_status != PERNIX_STATUS_OK) { EXPECT_TRUE(decompress_status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || - decompress_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || - decompress_status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + decompress_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || decompress_status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); return; } for (usize i = 0; i < input.size(); ++i) { - ASSERT_NEAR(backend_decompressed[i], fallback_restored[i], tolerance) - << "element=" << i << ", scale=" << scale; + ASSERT_NEAR(backend_decompressed[i], fallback_restored[i], tolerance) << "element=" << i << ", scale=" << scale; } } -} // namespace +} // namespace TEST(DeterministicCorrectnessTest, F32FallbackRoundTripsAllPatternsAndBitWidths) { for (u8 bit_width = pernix_min_bit_width(); bit_width <= pernix_max_bit_width(); ++bit_width) { @@ -275,7 +245,7 @@ TEST(DeterministicCorrectnessTest, SimdBackendsMatchFallbackWhereAvailable) { TEST(DeterministicCorrectnessTest, F32AndF64HaveConsistentFallbackSemantics) { constexpr u8 bit_width = 12; - const u32 elements = pernix_elements_per_block(bit_width) * 2U; + const u32 elements = pernix_elements_per_block(bit_width) * 2U; std::vector input_f32(elements); for (usize i = 0; i < input_f32.size(); ++i) { input_f32[i] = static_cast(static_cast(i % 9U) - 4); @@ -284,18 +254,14 @@ TEST(DeterministicCorrectnessTest, F32AndF64HaveConsistentFallbackSemantics) { std::vector compressed_f32(2 * kBlockSize); std::vector compressed_f64(2 * kBlockSize); - ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input_f32, 1.0f, compressed_f32, 2), - PERNIX_STATUS_OK); - ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input_f64, 1.0, compressed_f64, 2), - PERNIX_STATUS_OK); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input_f32, 1.0f, compressed_f32, 2), PERNIX_STATUS_OK); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input_f64, 1.0, compressed_f64, 2), PERNIX_STATUS_OK); EXPECT_EQ(compressed_f64, compressed_f32); std::vector restored_f32(input_f32.size()); std::vector restored_f64(input_f64.size()); - ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed_f32, 1.0f, restored_f32, 2), - PERNIX_STATUS_OK); - ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed_f64, 1.0, restored_f64, 2), - PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed_f32, 1.0f, restored_f32, 2), PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed_f64, 1.0, restored_f64, 2), PERNIX_STATUS_OK); for (usize i = 0; i < restored_f32.size(); ++i) { EXPECT_DOUBLE_EQ(static_cast(restored_f32[i]), restored_f64[i]); @@ -310,12 +276,12 @@ TEST(DeterministicCorrectnessTest, SignValuesFalseUsesUnsignedPackedValuesForAll std::vector signed_values(elements); std::vector unsigned_values(elements); - ASSERT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, - signed_values.data(), true), - PERNIX_STATUS_OK); - ASSERT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, - unsigned_values.data(), false), - PERNIX_STATUS_OK); + ASSERT_EQ( + pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, signed_values.data(), true), + PERNIX_STATUS_OK); + ASSERT_EQ( + pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, unsigned_values.data(), false), + PERNIX_STATUS_OK); const float expected_unsigned = static_cast((1U << bit_width) - 1U); EXPECT_EQ(unsigned_values[0], expected_unsigned); @@ -329,56 +295,51 @@ TEST(DeterministicCorrectnessTest, SignValuesFalseUsesUnsignedPackedValuesForAll TEST(DeterministicCorrectnessTest, CppSpanWrappersMatchCApiOutputAndStatus) { constexpr u8 bit_width = 10; - const auto input = make_dataset(Pattern::Increasing, bit_width, 1); - float scale = 0.0f; + const auto input = make_dataset(Pattern::Increasing, bit_width, 1); + float scale = 0.0f; ASSERT_EQ(compute_scale(max_abs(input), bit_width, scale), PERNIX_STATUS_OK); std::vector c_compressed(kBlockSize); std::vector cpp_compressed(kBlockSize); - EXPECT_EQ(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, input.data(), 1.0f / scale, - c_compressed.data()), + EXPECT_EQ(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, input.data(), 1.0f / scale, c_compressed.data()), PERNIX_STATUS_OK); - EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), - 1.0f / scale, std::span(cpp_compressed)), + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), 1.0f / scale, + std::span(cpp_compressed)), PERNIX_STATUS_OK); EXPECT_EQ(cpp_compressed, c_compressed); std::vector c_restored(input.size()); std::vector cpp_restored(input.size()); - EXPECT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, c_compressed.data(), scale, - c_restored.data(), true), - PERNIX_STATUS_OK); - EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, bit_width, kBlockSize, - std::span(cpp_compressed), scale, std::span(cpp_restored)), + EXPECT_EQ( + pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, c_compressed.data(), scale, c_restored.data(), true), + PERNIX_STATUS_OK); + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(cpp_compressed), scale, + std::span(cpp_restored)), PERNIX_STATUS_OK); EXPECT_EQ(cpp_restored, c_restored); - EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, 0, kBlockSize, std::span(input), - 1.0f / scale, std::span(cpp_compressed)), - pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, kBlockSize, input.data(), 1.0f / scale, - c_compressed.data())); + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, 0, kBlockSize, std::span(input), 1.0f / scale, + std::span(cpp_compressed)), + pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, kBlockSize, input.data(), 1.0f / scale, c_compressed.data())); } TEST(DeterministicCorrectnessTest, SpanValidationRejectsUndersizedMultiBlockBuffers) { constexpr u8 bit_width = 8; - const u32 elements = pernix_elements_per_block(bit_width) * 2U; + const u32 elements = pernix_elements_per_block(bit_width) * 2U; std::vector input(elements, 1.0f); std::vector compressed(2U * kBlockSize); std::vector restored(elements); EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, - std::span(input).first(input.size() - 1U), 1.0f, - std::span(compressed), 2), + std::span(input).first(input.size() - 1U), 1.0f, std::span(compressed), 2), PERNIX_STATUS_INVALID_ARGUMENT); - EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), - 1.0f, std::span(compressed).first(compressed.size() - 1U), 2), + EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), 1.0f, + std::span(compressed).first(compressed.size() - 1U), 2), PERNIX_STATUS_INVALID_ARGUMENT); EXPECT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, - std::span(compressed).first(compressed.size() - 1U), 1.0f, - std::span(restored), 2), + std::span(compressed).first(compressed.size() - 1U), 1.0f, std::span(restored), 2), PERNIX_STATUS_INVALID_ARGUMENT); - EXPECT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, - std::span(compressed), 1.0f, + EXPECT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(compressed), 1.0f, std::span(restored).first(restored.size() - 1U), 2), PERNIX_STATUS_INVALID_ARGUMENT); } diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp index 86840a2..587e12e 100644 --- a/tests/fallback_tests.cpp +++ b/tests/fallback_tests.cpp @@ -19,17 +19,14 @@ namespace { template -void expect_backend_compressed_block_matches_reference(FixtureT& fixture, const pernix_backend backend, - CompressFn compress_fn) { +void expect_backend_compressed_block_matches_reference(FixtureT& fixture, const pernix_backend backend, CompressFn compress_fn) { using ScaleType = std::remove_cvref_t; std::vector > compressed(fixture.testSet.numberOfBlocks); for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { compressed[b].resize(FixtureT::BlockSize); - const auto status = compress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getDecompressedData()[b].data(), - ScaleType{1} / fixture.testSet.getScales()[b], - compressed[b].data()); + const auto status = compress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[b].data(), + ScaleType{1} / fixture.testSet.getScales()[b], compressed[b].data()); ASSERT_EQ(status, PERNIX_STATUS_OK); } @@ -39,15 +36,13 @@ void expect_backend_compressed_block_matches_reference(FixtureT& fixture, const } template -void expect_backend_decompressed_block_matches_source(FixtureT& fixture, const pernix_backend backend, - DecompressFn decompress_fn) { +void expect_backend_decompressed_block_matches_source(FixtureT& fixture, const pernix_backend backend, DecompressFn decompress_fn) { using ValueType = std::remove_cvref_t; std::vector > decompressed(fixture.testSet.numberOfBlocks); for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { decompressed[b].resize(fixture.testSet.elementsPerBlock); - const auto status = decompress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getCompressedData()[b].data(), + const auto status = decompress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[b].data(), fixture.testSet.getScales()[b], decompressed[b].data(), true); ASSERT_EQ(status, PERNIX_STATUS_OK); } @@ -56,7 +51,7 @@ void expect_backend_decompressed_block_matches_source(FixtureT& fixture, const p expectDecompressedBlockNearSource(fixture, decompressed[b], b); } } -} +} // namespace // --------------------------------------------------------------------------- // Fallback compress: verify byte-exact match against the reference @@ -68,10 +63,9 @@ TYPED_TEST(CompressionTest, FallbackCompressBlock) { for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { compressed[b].resize(TestFixture::BlockSize); - const auto status = pernix_compress_block_f32( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - this->testSet.getDecompressedData()[b].data(), 1.0f / this->testSet.getScales()[b], - compressed[b].data()); + const auto status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getDecompressedData()[b].data(), 1.0f / this->testSet.getScales()[b], + compressed[b].data()); ASSERT_EQ(status, PERNIX_STATUS_OK); } @@ -86,10 +80,9 @@ TYPED_TEST(CompressionTest64, FallbackCompressBlock) { for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { compressed[b].resize(TestFixture::BlockSize); - const auto status = pernix_compress_block_f64( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - this->testSet.getDecompressedData()[b].data(), 1.0 / this->testSet.getScales()[b], - compressed[b].data()); + const auto status = pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getDecompressedData()[b].data(), 1.0 / this->testSet.getScales()[b], + compressed[b].data()); ASSERT_EQ(status, PERNIX_STATUS_OK); } @@ -108,10 +101,9 @@ TYPED_TEST(DecompressionTest, FallbackDecompressBlock) { for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { decompressed[b].resize(this->testSet.elementsPerBlock); - const auto status = pernix_decompress_block_f32( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - this->testSet.getCompressedData()[b].data(), this->testSet.getScales()[b], - decompressed[b].data(), true); + const auto status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getCompressedData()[b].data(), this->testSet.getScales()[b], + decompressed[b].data(), true); ASSERT_EQ(status, PERNIX_STATUS_OK); } @@ -126,10 +118,9 @@ TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { decompressed[b].resize(this->testSet.elementsPerBlock); - const auto status = pernix_decompress_block_f64( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - this->testSet.getCompressedData()[b].data(), this->testSet.getScales()[b], - decompressed[b].data(), true); + const auto status = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getCompressedData()[b].data(), this->testSet.getScales()[b], + decompressed[b].data(), true); ASSERT_EQ(status, PERNIX_STATUS_OK); } @@ -140,23 +131,19 @@ TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) TYPED_TEST(CompressionTest, FallbackStdparCompressBlock) { - expect_backend_compressed_block_matches_reference(*this, PERNIX_BACKEND_FALLBACK_STDPAR, - &pernix_compress_block_f32); + expect_backend_compressed_block_matches_reference(*this, PERNIX_BACKEND_FALLBACK_STDPAR, &pernix_compress_block_f32); } TYPED_TEST(CompressionTest64, FallbackStdparCompressBlock) { - expect_backend_compressed_block_matches_reference(*this, PERNIX_BACKEND_FALLBACK_STDPAR, - &pernix_compress_block_f64); + expect_backend_compressed_block_matches_reference(*this, PERNIX_BACKEND_FALLBACK_STDPAR, &pernix_compress_block_f64); } TYPED_TEST(DecompressionTest, FallbackStdparDecompressBlock) { - expect_backend_decompressed_block_matches_source(*this, PERNIX_BACKEND_FALLBACK_STDPAR, - &pernix_decompress_block_f32); + expect_backend_decompressed_block_matches_source(*this, PERNIX_BACKEND_FALLBACK_STDPAR, &pernix_decompress_block_f32); } TYPED_TEST(DecompressionTest64, FallbackStdparDecompressBlock) { - expect_backend_decompressed_block_matches_source(*this, PERNIX_BACKEND_FALLBACK_STDPAR, - &pernix_decompress_block_f64); + expect_backend_decompressed_block_matches_source(*this, PERNIX_BACKEND_FALLBACK_STDPAR, &pernix_decompress_block_f64); } #endif @@ -165,14 +152,13 @@ TYPED_TEST(DecompressionTest64, FallbackStdparDecompressBlock) { // --------------------------------------------------------------------------- TYPED_TEST(CompressionTest, FallbackCompressBlocksRoundtrip) { - const u32 nb = this->testSet.numberOfBlocks; - const u32 epb = this->testSet.elementsPerBlock; + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; const u32 total = nb * epb; std::vector flat(total); for (u32 b = 0; b < nb; b++) { - std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, - flat.data() + b * epb); + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); } // Compute a single scale that covers all blocks @@ -180,20 +166,18 @@ TYPED_TEST(CompressionTest, FallbackCompressBlocksRoundtrip) { for (u32 i = 0; i < total; i++) { max_abs = std::max(max_abs, std::abs(flat[i])); } - const float q = static_cast(decltype(this->testSet)::quantization_levels); - const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); + const float q = static_cast(decltype(this->testSet)::quantization_levels); + const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); const float scale_inv = 1.0f / scale; std::vector compressed(nb * TestFixture::BlockSize); - auto status = pernix_compress_blocks_f32( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - flat.data(), scale_inv, compressed.data(), nb); + auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), scale_inv, + compressed.data(), nb); ASSERT_EQ(status, PERNIX_STATUS_OK); std::vector restored(total); - status = pernix_decompress_blocks_f32( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - compressed.data(), scale, restored.data(), nb, true); + status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, + restored.data(), nb, true); ASSERT_EQ(status, PERNIX_STATUS_OK); const float tol = (std::abs(scale) * 0.5f) + (std::numeric_limits::epsilon() * 16.0f); @@ -203,14 +187,13 @@ TYPED_TEST(CompressionTest, FallbackCompressBlocksRoundtrip) { } TYPED_TEST(CompressionTest64, FallbackCompressBlocksRoundtrip) { - const u32 nb = this->testSet.numberOfBlocks; - const u32 epb = this->testSet.elementsPerBlock; + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; const u32 total = nb * epb; std::vector flat(total); for (u32 b = 0; b < nb; b++) { - std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, - flat.data() + b * epb); + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); } // Compute a single scale that covers all blocks @@ -218,20 +201,18 @@ TYPED_TEST(CompressionTest64, FallbackCompressBlocksRoundtrip) { for (u32 i = 0; i < total; i++) { max_abs = std::max(max_abs, std::abs(flat[i])); } - const double q = static_cast(decltype(this->testSet)::quantization_levels); - const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); + const double q = static_cast(decltype(this->testSet)::quantization_levels); + const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); const double scale_inv = 1.0 / scale; std::vector compressed(nb * TestFixture::BlockSize); - auto status = pernix_compress_blocks_f64( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - flat.data(), scale_inv, compressed.data(), nb); + auto status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), scale_inv, + compressed.data(), nb); ASSERT_EQ(status, PERNIX_STATUS_OK); std::vector restored(total); - status = pernix_decompress_blocks_f64( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - compressed.data(), scale, restored.data(), nb, true); + status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, + restored.data(), nb, true); ASSERT_EQ(status, PERNIX_STATUS_OK); const double tol = (std::abs(scale) * 0.5) + (std::numeric_limits::epsilon() * 16.0); @@ -242,8 +223,8 @@ TYPED_TEST(CompressionTest64, FallbackCompressBlocksRoundtrip) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) TYPED_TEST(CompressionTest, FallbackStdparCompressBlocksRoundtrip) { - const u32 nb = this->testSet.numberOfBlocks; - const u32 epb = this->testSet.elementsPerBlock; + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; const u32 total = nb * epb; std::vector flat(total); @@ -255,18 +236,18 @@ TYPED_TEST(CompressionTest, FallbackStdparCompressBlocksRoundtrip) { for (u32 i = 0; i < total; ++i) { max_abs = std::max(max_abs, std::abs(flat[i])); } - const float q = static_cast(decltype(this->testSet)::quantization_levels); - const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); + const float q = static_cast(decltype(this->testSet)::quantization_levels); + const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); const float scale_inv = 1.0f / scale; std::vector compressed(nb * TestFixture::BlockSize); - auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, - TestFixture::BlockSize, flat.data(), scale_inv, compressed.data(), nb); + auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), + scale_inv, compressed.data(), nb); ASSERT_EQ(status, PERNIX_STATUS_OK); std::vector restored(total); - status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, - TestFixture::BlockSize, compressed.data(), scale, restored.data(), nb, true); + status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), + scale, restored.data(), nb, true); ASSERT_EQ(status, PERNIX_STATUS_OK); const float tol = (std::abs(scale) * 0.5f) + (std::numeric_limits::epsilon() * 16.0f); @@ -276,8 +257,8 @@ TYPED_TEST(CompressionTest, FallbackStdparCompressBlocksRoundtrip) { } TYPED_TEST(CompressionTest64, FallbackStdparCompressBlocksRoundtrip) { - const u32 nb = this->testSet.numberOfBlocks; - const u32 epb = this->testSet.elementsPerBlock; + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; const u32 total = nb * epb; std::vector flat(total); @@ -289,18 +270,18 @@ TYPED_TEST(CompressionTest64, FallbackStdparCompressBlocksRoundtrip) { for (u32 i = 0; i < total; ++i) { max_abs = std::max(max_abs, std::abs(flat[i])); } - const double q = static_cast(decltype(this->testSet)::quantization_levels); - const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); + const double q = static_cast(decltype(this->testSet)::quantization_levels); + const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); const double scale_inv = 1.0 / scale; std::vector compressed(nb * TestFixture::BlockSize); - auto status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, - TestFixture::BlockSize, flat.data(), scale_inv, compressed.data(), nb); + auto status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), + scale_inv, compressed.data(), nb); ASSERT_EQ(status, PERNIX_STATUS_OK); std::vector restored(total); - status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, - TestFixture::BlockSize, compressed.data(), scale, restored.data(), nb, true); + status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), + scale, restored.data(), nb, true); ASSERT_EQ(status, PERNIX_STATUS_OK); const double tol = (std::abs(scale) * 0.5) + (std::numeric_limits::epsilon() * 16.0); @@ -315,20 +296,18 @@ TYPED_TEST(CompressionTest64, FallbackStdparCompressBlocksRoundtrip) { // --------------------------------------------------------------------------- TYPED_TEST(CompressionTest, SingleBlockCompressBlocksMatchesBlock) { - const auto &src = this->testSet.getDecompressedData()[0]; + const auto& src = this->testSet.getDecompressedData()[0]; const float scale_inv = 1.0f / this->testSet.getScales()[0]; std::vector blockOut(TestFixture::BlockSize); std::vector blocksOut(TestFixture::BlockSize); - auto s1 = pernix_compress_block_f32( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - src.data(), scale_inv, blockOut.data()); + auto s1 = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, src.data(), scale_inv, + blockOut.data()); ASSERT_EQ(s1, PERNIX_STATUS_OK); - auto s2 = pernix_compress_blocks_f32( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - src.data(), scale_inv, blocksOut.data(), 1); + auto s2 = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, src.data(), scale_inv, + blocksOut.data(), 1); ASSERT_EQ(s2, PERNIX_STATUS_OK); for (u32 i = 0; i < TestFixture::BlockSize; i++) { @@ -337,21 +316,19 @@ TYPED_TEST(CompressionTest, SingleBlockCompressBlocksMatchesBlock) { } TYPED_TEST(DecompressionTest, SingleBlockDecompressBlocksMatchesBlock) { - const auto &compressed = this->testSet.getCompressedData()[0]; - const float scale = this->testSet.getScales()[0]; - const u32 epb = this->testSet.elementsPerBlock; + const auto& compressed = this->testSet.getCompressedData()[0]; + const float scale = this->testSet.getScales()[0]; + const u32 epb = this->testSet.elementsPerBlock; std::vector blockOut(epb); std::vector blocksOut(epb); - auto s1 = pernix_decompress_block_f32( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - compressed.data(), scale, blockOut.data(), true); + auto s1 = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, + blockOut.data(), true); ASSERT_EQ(s1, PERNIX_STATUS_OK); - auto s2 = pernix_decompress_blocks_f32( - PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, - compressed.data(), scale, blocksOut.data(), 1, true); + auto s2 = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, + blocksOut.data(), 1, true); ASSERT_EQ(s2, PERNIX_STATUS_OK); for (u32 i = 0; i < epb; i++) { @@ -361,18 +338,18 @@ TYPED_TEST(DecompressionTest, SingleBlockDecompressBlocksMatchesBlock) { #if defined(PERNIX_BUILD_FALLBACK_STDPAR) TYPED_TEST(CompressionTest, SingleBlockStdparCompressBlocksMatchesBlock) { - const auto& src = this->testSet.getDecompressedData()[0]; + const auto& src = this->testSet.getDecompressedData()[0]; const float scale_inv = 1.0f / this->testSet.getScales()[0]; std::vector block_out(TestFixture::BlockSize); std::vector blocks_out(TestFixture::BlockSize); - auto s1 = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, - TestFixture::BlockSize, src.data(), scale_inv, block_out.data()); + auto s1 = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, src.data(), + scale_inv, block_out.data()); ASSERT_EQ(s1, PERNIX_STATUS_OK); - auto s2 = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, - TestFixture::BlockSize, src.data(), scale_inv, blocks_out.data(), 1); + auto s2 = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, src.data(), + scale_inv, blocks_out.data(), 1); ASSERT_EQ(s2, PERNIX_STATUS_OK); for (u32 i = 0; i < TestFixture::BlockSize; ++i) { @@ -382,19 +359,18 @@ TYPED_TEST(CompressionTest, SingleBlockStdparCompressBlocksMatchesBlock) { TYPED_TEST(DecompressionTest, SingleBlockStdparDecompressBlocksMatchesBlock) { const auto& compressed = this->testSet.getCompressedData()[0]; - const float scale = this->testSet.getScales()[0]; - const u32 epb = this->testSet.elementsPerBlock; + const float scale = this->testSet.getScales()[0]; + const u32 epb = this->testSet.elementsPerBlock; std::vector block_out(epb); std::vector blocks_out(epb); - auto s1 = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, - TestFixture::BlockSize, compressed.data(), scale, block_out.data(), true); + auto s1 = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), + scale, block_out.data(), true); ASSERT_EQ(s1, PERNIX_STATUS_OK); - auto s2 = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, - TestFixture::BlockSize, compressed.data(), scale, blocks_out.data(), 1, - true); + auto s2 = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), + scale, blocks_out.data(), 1, true); ASSERT_EQ(s2, PERNIX_STATUS_OK); for (u32 i = 0; i < epb; ++i) { @@ -420,8 +396,8 @@ TEST(FallbackEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { } TEST(FallbackEdgeTest, ClearsUnusedPaddingBytes) { - constexpr u32 BS = 64; - constexpr u32 BW = 24; + constexpr u32 BS = 64; + constexpr u32 BW = 24; constexpr u32 EPB = (BS * 8) / BW; std::array input{}; @@ -434,8 +410,8 @@ TEST(FallbackEdgeTest, ClearsUnusedPaddingBytes) { } TEST(FallbackEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { - constexpr u32 BS = 64; - constexpr u32 BW = 4; + constexpr u32 BS = 64; + constexpr u32 BW = 4; constexpr u32 EPB = (BS * 8) / BW; std::array input{}; @@ -464,11 +440,9 @@ TEST(FallbackEdgeTest, SignValuesFalseTreatsPackedValuesAsUnsigned) { std::array signed_output{}; std::array unsigned_output{}; - auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, signed_output.data(), - true); + auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, signed_output.data(), true); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, unsigned_output.data(), - false); + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, unsigned_output.data(), false); ASSERT_EQ(st, PERNIX_STATUS_OK); EXPECT_EQ(signed_output[0], -1.0f); @@ -476,11 +450,11 @@ TEST(FallbackEdgeTest, SignValuesFalseTreatsPackedValuesAsUnsigned) { } TEST(FallbackEdgeTest, FixedBlockInvariantAndMixedSignMultiBlockRoundTrip) { - constexpr u32 BS = 64; - constexpr u32 BW = 12; + constexpr u32 BS = 64; + constexpr u32 BW = 12; constexpr u32 blocks = 3; - constexpr u32 EPB = (BS * 8) / BW; - constexpr u32 total = EPB * blocks; + constexpr u32 EPB = (BS * 8) / BW; + constexpr u32 total = EPB * blocks; static_assert(BS == 64); static_assert(EPB == 42); @@ -491,8 +465,8 @@ TEST(FallbackEdgeTest, FixedBlockInvariantAndMixedSignMultiBlockRoundTrip) { std::array input_f64{}; for (u32 i = 0; i < total; ++i) { const auto centered = static_cast(i % 31U) - 15; - input_f32[i] = static_cast(centered) * 0.25f; - input_f64[i] = static_cast(centered) * 0.25; + input_f32[i] = static_cast(centered) * 0.25f; + input_f64[i] = static_cast(centered) * 0.25; } std::array compressed_f32{}; @@ -500,18 +474,14 @@ TEST(FallbackEdgeTest, FixedBlockInvariantAndMixedSignMultiBlockRoundTrip) { std::array restored_f32{}; std::array restored_f64{}; - auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input_f32.data(), 4.0f, - compressed_f32.data(), blocks); + auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input_f32.data(), 4.0f, compressed_f32.data(), blocks); ASSERT_EQ(status, PERNIX_STATUS_OK); - status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed_f32.data(), 0.25f, - restored_f32.data(), blocks, true); + status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed_f32.data(), 0.25f, restored_f32.data(), blocks, true); ASSERT_EQ(status, PERNIX_STATUS_OK); - status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input_f64.data(), 4.0, - compressed_f64.data(), blocks); + status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input_f64.data(), 4.0, compressed_f64.data(), blocks); ASSERT_EQ(status, PERNIX_STATUS_OK); - status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed_f64.data(), 0.25, - restored_f64.data(), blocks, true); + status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed_f64.data(), 0.25, restored_f64.data(), blocks, true); ASSERT_EQ(status, PERNIX_STATUS_OK); for (u32 i = 0; i < total; ++i) { @@ -520,14 +490,13 @@ TEST(FallbackEdgeTest, FixedBlockInvariantAndMixedSignMultiBlockRoundTrip) { } std::array unsigned_output{}; - status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, compressed_f32.data(), 1.0f, - unsigned_output.data(), false); + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, compressed_f32.data(), 1.0f, unsigned_output.data(), false); ASSERT_EQ(status, PERNIX_STATUS_OK); } TEST(FallbackEdgeTest, CApiRejectsInvalidScale) { - constexpr u32 BS = 64; - constexpr u32 BW = 8; + constexpr u32 BS = 64; + constexpr u32 BW = 8; constexpr u32 EPB = (BS * 8) / BW; std::array input{}; @@ -536,17 +505,16 @@ TEST(FallbackEdgeTest, CApiRejectsInvalidScale) { EXPECT_EQ(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 0.0f, compressed.data()), PERNIX_STATUS_INVALID_ARGUMENT); - EXPECT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), -1.0f, output.data(), - true), - PERNIX_STATUS_INVALID_ARGUMENT); - EXPECT_EQ(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), - std::numeric_limits::infinity(), compressed.data()), + EXPECT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), -1.0f, output.data(), true), PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ( + pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), std::numeric_limits::infinity(), compressed.data()), + PERNIX_STATUS_INVALID_ARGUMENT); } TEST(FallbackEdgeTest, CApiRejectsInvalidScaleF64) { - constexpr u32 BS = 64; - constexpr u32 BW = 8; + constexpr u32 BS = 64; + constexpr u32 BW = 8; constexpr u32 EPB = (BS * 8) / BW; std::array input{}; @@ -555,13 +523,12 @@ TEST(FallbackEdgeTest, CApiRejectsInvalidScaleF64) { EXPECT_EQ(pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 0.0, compressed.data()), PERNIX_STATUS_INVALID_ARGUMENT); - EXPECT_EQ(pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), -1.0, output.data(), - true), + EXPECT_EQ(pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), -1.0, output.data(), true), PERNIX_STATUS_INVALID_ARGUMENT); } TEST(FallbackEdgeTest, ScaleHelpersValidateInputs) { - float scale_f32 = 0.0f; + float scale_f32 = 0.0f; double scale_f64 = 0.0; EXPECT_EQ(pernix_scale_f32(32767.0f, 16, &scale_f32), PERNIX_STATUS_OK); @@ -570,8 +537,7 @@ TEST(FallbackEdgeTest, ScaleHelpersValidateInputs) { EXPECT_DOUBLE_EQ(scale_f64, 1.0); EXPECT_EQ(pernix_scale_f32(1.0f, 0, &scale_f32), PERNIX_STATUS_INVALID_ARGUMENT); EXPECT_EQ(pernix_scale_f32(-1.0f, 16, &scale_f32), PERNIX_STATUS_INVALID_ARGUMENT); - EXPECT_EQ(pernix_scale_f64(std::numeric_limits::infinity(), 16, &scale_f64), - PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_scale_f64(std::numeric_limits::infinity(), 16, &scale_f64), PERNIX_STATUS_INVALID_ARGUMENT); EXPECT_EQ(pernix_scale_f64(1.0, 16, nullptr), PERNIX_STATUS_INVALID_ARGUMENT); } @@ -581,30 +547,28 @@ TEST(FallbackStdparEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { const std::array input{0x08}; std::array output{}; - const auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, 4, BS, input.data(), 1.0f, - output.data(), true); + const auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, 4, BS, input.data(), 1.0f, output.data(), true); ASSERT_EQ(st, PERNIX_STATUS_OK); EXPECT_EQ(output[0], -8.0f); } TEST(FallbackStdparEdgeTest, ClearsUnusedPaddingBytes) { - constexpr u32 BS = 64; - constexpr u32 BW = 24; + constexpr u32 BS = 64; + constexpr u32 BW = 24; constexpr u32 EPB = (BS * 8) / BW; std::array input{}; std::array output{}; output.fill(0xA5); - const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, - output.data()); + const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, output.data()); ASSERT_EQ(st, PERNIX_STATUS_OK); EXPECT_EQ(output[BS - 1], 0); } TEST(FallbackStdparEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { - constexpr u32 BS = 64; - constexpr u32 BW = 4; + constexpr u32 BS = 64; + constexpr u32 BW = 4; constexpr u32 EPB = (BS * 8) / BW; std::array input{}; @@ -615,12 +579,10 @@ TEST(FallbackStdparEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { std::array compressed{}; std::array restored{}; - auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, - compressed.data()); + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, compressed.data()); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, - restored.data(), true); + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, restored.data(), true); ASSERT_EQ(st, PERNIX_STATUS_OK); EXPECT_EQ(restored[0], 7.0f); @@ -648,8 +610,7 @@ TEST(FallbackStdparEdgeTest, TailWidthCompressMatchesScalarF32) { st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, stdpar_block.data()); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, - stdpar_blocks.data(), 1); + st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, stdpar_blocks.data(), 1); ASSERT_EQ(st, PERNIX_STATUS_OK); for (u32 i = 0; i < BS; ++i) { @@ -678,8 +639,7 @@ TEST(FallbackStdparEdgeTest, TailWidthCompressMatchesScalarF64) { st = pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0, stdpar_block.data()); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0, - stdpar_blocks.data(), 1); + st = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0, stdpar_blocks.data(), 1); ASSERT_EQ(st, PERNIX_STATUS_OK); for (u32 i = 0; i < BS; ++i) { @@ -709,12 +669,10 @@ TEST(FallbackStdparEdgeTest, TailWidthDecompressMatchesScalarF32) { st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), 1.0f, scalar_block.data(), true); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, - stdpar_block.data(), true); + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, stdpar_block.data(), true); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, - stdpar_blocks.data(), 1, true); + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, stdpar_blocks.data(), 1, true); ASSERT_EQ(st, PERNIX_STATUS_OK); for (u32 i = 0; i < EPB; ++i) { @@ -744,12 +702,10 @@ TEST(FallbackStdparEdgeTest, TailWidthDecompressMatchesScalarF64) { st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), 1.0, scalar_block.data(), true); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0, - stdpar_block.data(), true); + st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0, stdpar_block.data(), true); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0, - stdpar_blocks.data(), 1, true); + st = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0, stdpar_blocks.data(), 1, true); ASSERT_EQ(st, PERNIX_STATUS_OK); for (u32 i = 0; i < EPB; ++i) { @@ -765,8 +721,8 @@ TEST(FallbackStdparEdgeTest, TailWidthDecompressMatchesScalarF64) { TEST(ErrorCodeTest, UnsupportedBlockSizeReturnsError) { constexpr u32 BS = 32; - f32 src[32] = {}; - u8 dst[32] = {}; + f32 src[32] = {}; + u8 dst[32] = {}; auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, BS, src, 1.0f, dst); EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); @@ -783,8 +739,8 @@ TEST(ErrorCodeTest, UnsupportedBlockSizeReturnsError) { TEST(ErrorCodeTest, UnsupportedBitWidthReturnsError) { constexpr u32 BS = 64; - f32 src[256] = {}; - u8 dst[64] = {}; + f32 src[256] = {}; + u8 dst[64] = {}; auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, BS, src, 1.0f, dst); EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); @@ -795,7 +751,7 @@ TEST(ErrorCodeTest, UnsupportedBitWidthReturnsError) { TEST(ErrorCodeTest, NullPointerReturnsError) { f32 src[64] = {}; - u8 dst[64] = {}; + u8 dst[64] = {}; auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, 64, nullptr, 1.0f, dst); EXPECT_EQ(st, PERNIX_STATUS_INVALID_ARGUMENT); @@ -810,10 +766,10 @@ TEST(ErrorCodeTest, FallbackAliasMatchesScalar) { TEST(ErrorCodeTest, FallbackStdparReturnsUnsupportedImplementationByDefault) { constexpr u32 BS = 64; - constexpr u8 BW = 8; + constexpr u8 BW = 8; f32 src[(BS * 8U) / BW] = {}; - u8 dst[BS] = {}; + u8 dst[BS] = {}; #if defined(PERNIX_BUILD_FALLBACK_STDPAR) const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, src, 1.0f, dst); @@ -826,10 +782,10 @@ TEST(ErrorCodeTest, FallbackStdparReturnsUnsupportedImplementationByDefault) { TEST(ErrorCodeTest, FallbackSimdReturnsUnsupportedImplementationByDefault) { constexpr u32 BS = 64; - constexpr u8 BW = 8; + constexpr u8 BW = 8; f32 src[(BS * 8U) / BW] = {}; - u8 dst[BS] = {}; + u8 dst[BS] = {}; const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_SIMD, BW, BS, src, 1.0f, dst); EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); @@ -837,24 +793,24 @@ TEST(ErrorCodeTest, FallbackSimdReturnsUnsupportedImplementationByDefault) { TEST(ErrorCodeTest, ExplicitUnavailableBackendReturnsError) { constexpr u32 BS = 64; - constexpr u8 BW = 8; + constexpr u8 BW = 8; f32 src[(BS * 8U) / BW] = {}; - u8 dst[BS] = {}; + u8 dst[BS] = {}; pernix_backend backend = PERNIX_BACKEND_FALLBACK; - bool found = false; + bool found = false; #if defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) if (!__builtin_cpu_supports("avx512vbmi")) { backend = PERNIX_BACKEND_X86_AVX512_VBMI; - found = true; + found = true; } else if (!(__builtin_cpu_supports("avx2") && __builtin_cpu_supports("bmi2"))) { backend = PERNIX_BACKEND_X86_BMI2; - found = true; + found = true; } else if (!__builtin_cpu_supports("avx2")) { backend = PERNIX_BACKEND_X86_AVX2; - found = true; + found = true; } #elif defined(__aarch64__) || defined(_M_ARM64) const bool neon = true; @@ -865,10 +821,10 @@ TEST(ErrorCodeTest, ExplicitUnavailableBackendReturnsError) { #endif if (!sve2) { backend = PERNIX_BACKEND_ARM64_SVE; - found = true; + found = true; } else if (!neon) { backend = PERNIX_BACKEND_ARM64_NEON; - found = true; + found = true; } #endif @@ -878,6 +834,5 @@ TEST(ErrorCodeTest, ExplicitUnavailableBackendReturnsError) { const auto st = pernix_compress_block_f32(backend, BW, BS, src, 1.0f, dst); // A backend can be absent from the build, or compiled but unsupported by this CPU. - EXPECT_TRUE(st == PERNIX_STATUS_UNSUPPORTED_BACKEND || - st == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); + EXPECT_TRUE(st == PERNIX_STATUS_UNSUPPORTED_BACKEND || st == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); } diff --git a/tests/header_only_tests.cpp b/tests/header_only_tests.cpp index ca58e16..52ac3e1 100644 --- a/tests/header_only_tests.cpp +++ b/tests/header_only_tests.cpp @@ -1,14 +1,13 @@ #include -#include - #include #include +#include #include namespace { -constexpr u8 kBitWidth = 8; -constexpr u32 kBlockSize = PERNIX_TEST_BLOCK_SIZE; +constexpr u8 kBitWidth = 8; +constexpr u32 kBlockSize = PERNIX_TEST_BLOCK_SIZE; constexpr u32 kBlockElements = (kBlockSize * 8U) / kBitWidth; template @@ -26,18 +25,18 @@ void expect_round_trip(const pernix::Backend backend) { std::vector compressed(kBlockSize); std::vector restored(kBlockElements, static_cast(0)); - ASSERT_EQ(pernix::compress_block(backend, kBitWidth, kBlockSize, std::span(input), - static_cast(1), std::span(compressed)), + ASSERT_EQ(pernix::compress_block(backend, kBitWidth, kBlockSize, std::span(input), static_cast(1), + std::span(compressed)), PERNIX_STATUS_OK); - ASSERT_EQ(pernix::decompress_block(backend, kBitWidth, kBlockSize, std::span(compressed), - static_cast(1), std::span(restored)), + ASSERT_EQ(pernix::decompress_block(backend, kBitWidth, kBlockSize, std::span(compressed), static_cast(1), + std::span(restored)), PERNIX_STATUS_OK); for (usize i = 0; i < restored.size(); ++i) { EXPECT_NEAR(restored[i], input[i], static_cast(1.0)); } } -} +} // namespace TEST(HeaderOnlyPernix, FallbackFloatRoundTrip) { expect_round_trip(pernix::Backend::Fallback); @@ -67,8 +66,8 @@ TEST(HeaderOnlyPernix, PublicHelpersDescribeFixedBlockFormat) { TEST(HeaderOnlyPernix, ScaleHelpersNameCompressionAndDecompressionConventions) { float decompression_scale = 0.0f; - float compression_scale = 0.0f; - float inverse_scale = 0.0f; + float compression_scale = 0.0f; + float inverse_scale = 0.0f; EXPECT_EQ(pernix::decompression_scale_from_bmax(127.0f, 8, decompression_scale), PERNIX_STATUS_OK); EXPECT_FLOAT_EQ(decompression_scale, 1.0f); @@ -85,8 +84,7 @@ TEST(HeaderOnlyPernix, StatusStringsArePublic) { pernix::Status status = PERNIX_STATUS_OK; EXPECT_EQ(status, PERNIX_STATUS_OK); EXPECT_STREQ(pernix::status_string(status), "PERNIX_STATUS_OK"); - EXPECT_STREQ(pernix::status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION), - "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION"); + EXPECT_STREQ(pernix::status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION), "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION"); EXPECT_STREQ(pernix::status_string(static_cast(123)), "PERNIX_STATUS_UNKNOWN"); } @@ -94,8 +92,8 @@ TEST(HeaderOnlyPernix, RejectsUndersizedCompressInputSpan) { std::vector input(kBlockElements - 1U, 0.0f); std::vector output(kBlockSize); - EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), - 1.0f, std::span(output)), + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(output)), PERNIX_STATUS_INVALID_ARGUMENT); } @@ -103,22 +101,22 @@ TEST(HeaderOnlyPernix, RejectsUndersizedCompressOutputSpan) { std::vector input(kBlockElements, 0.0f); std::vector output(kBlockSize - 1U); - EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), - 1.0f, std::span(output)), + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(output)), PERNIX_STATUS_INVALID_ARGUMENT); } TEST(HeaderOnlyPernix, RejectsUndersizedDecompressSpans) { std::vector short_input(kBlockSize - 1U); std::vector output(kBlockElements); - EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, - std::span(short_input), 1.0f, std::span(output)), + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(short_input), 1.0f, + std::span(output)), PERNIX_STATUS_INVALID_ARGUMENT); std::vector input(kBlockSize); std::vector short_output(kBlockElements - 1U); - EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), - 1.0f, std::span(short_output)), + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(short_output)), PERNIX_STATUS_INVALID_ARGUMENT); } @@ -126,14 +124,12 @@ TEST(HeaderOnlyPernix, RejectsInvalidSpanParametersBeforeDispatch) { std::vector input(kBlockElements, 0.0f); std::vector output(kBlockSize); - EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, 0, kBlockSize, std::span(input), 1.0f, - std::span(output)), + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, 0, kBlockSize, std::span(input), 1.0f, std::span(output)), PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); - EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, 96, std::span(input), 1.0f, - std::span(output)), + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, 96, std::span(input), 1.0f, std::span(output)), PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); - EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, kBitWidth, kBlockSize, - std::span(input), 1.0f, std::span(output), 0), + EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(output), 0), PERNIX_STATUS_INVALID_ARGUMENT); } @@ -142,11 +138,11 @@ TEST(HeaderOnlyPernix, RejectsInvalidScale) { std::vector compressed(kBlockSize); std::vector output(input.size()); - EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), - 0.0f, std::span(compressed)), + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 0.0f, + std::span(compressed)), PERNIX_STATUS_INVALID_ARGUMENT); - EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, - std::span(compressed), -1.0f, std::span(output)), + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(compressed), -1.0f, + std::span(output)), PERNIX_STATUS_INVALID_ARGUMENT); EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), std::numeric_limits::infinity(), std::span(compressed)), @@ -162,12 +158,12 @@ TEST(HeaderOnlyPernix, FallbackStdparReturnsUnsupportedImplementation) { std::vector compressed(kBlockSize); #if defined(PERNIX_BUILD_FALLBACK_STDPAR) - EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, kBitWidth, kBlockSize, - std::span(input), 1.0f, std::span(compressed)), + EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(compressed)), PERNIX_STATUS_OK); #else - EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, kBitWidth, kBlockSize, - std::span(input), 1.0f, std::span(compressed)), + EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(compressed)), PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); #endif } @@ -176,8 +172,8 @@ TEST(HeaderOnlyPernix, FallbackSimdReturnsUnsupportedImplementation) { auto input = make_input(); std::vector compressed(kBlockSize); - EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackSimd, kBitWidth, kBlockSize, - std::span(input), 1.0f, std::span(compressed)), + EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackSimd, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(compressed)), PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); } diff --git a/tests/include/testset.h b/tests/include/testset.h index b73fc71..392ed44 100644 --- a/tests/include/testset.h +++ b/tests/include/testset.h @@ -1,13 +1,13 @@ #ifndef PERNIX_TESTSET_H #define PERNIX_TESTSET_H -#include #include +#include #include -#include -#include #include +#include +#include #include #include #include @@ -18,14 +18,13 @@ #define PERNIX_TEST_BLOCK_SIZE 64 #endif -static_assert(PERNIX_TEST_BLOCK_SIZE % 32 == 0, - "PERNIX_TEST_BLOCK_SIZE must be dividable by 32 bytes"); +static_assert(PERNIX_TEST_BLOCK_SIZE % 32 == 0, "PERNIX_TEST_BLOCK_SIZE must be dividable by 32 bytes"); -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && std::is_floating_point_v class TestSet { using ValueType = u8; - using SeedType = std::mt19937::result_type; + using SeedType = std::mt19937::result_type; alignas(64) std::vector > compressedData; alignas(64) std::vector > sourceData; @@ -40,17 +39,14 @@ class TestSet { static constexpr SeedType defaultSeed = 0x5eed1234u; static constexpr T quantization_levels = - SIGN_VALUES - ? static_cast(BIT_WIDTH == 1 ? 1u : ((1u << (BIT_WIDTH - 1u)) - 1u)) - : static_cast((1u << BIT_WIDTH) - 1u); + SIGN_VALUES ? static_cast(BIT_WIDTH == 1 ? 1u : ((1u << (BIT_WIDTH - 1u)) - 1u)) : static_cast((1u << BIT_WIDTH) - 1u); u32 numberOfBlocks; [[nodiscard]] constexpr u32 totalElements() const { return numberOfBlocks * elementsPerBlock; } [[nodiscard]] T blockTolerance(const u32 block) const { - return (std::abs(scalesData[block]) * static_cast(0.5)) + ( - std::numeric_limits::epsilon() * static_cast(16)); + return (std::abs(scalesData[block]) * static_cast(0.5)) + (std::numeric_limits::epsilon() * static_cast(16)); } explicit TestSet(const u32 number_of_blocks, const SeedType initial_seed = testSeed()) @@ -62,21 +58,21 @@ class TestSet { generateData(); } - [[nodiscard]] const std::vector &getScales() const { return scalesData; } + [[nodiscard]] const std::vector& getScales() const { return scalesData; } - [[nodiscard]] const std::vector > &getCompressedData() const { return compressedData; } + [[nodiscard]] const std::vector >& getCompressedData() const { return compressedData; } - [[nodiscard]] const std::vector > &getDecompressedData() const { return sourceData; } + [[nodiscard]] const std::vector >& getDecompressedData() const { return sourceData; } [[nodiscard]] SeedType getSeed() const { return seed; } [[nodiscard]] static SeedType testSeed() { - const char *env_seed = std::getenv("PERNIX_TEST_SEED"); + const char* env_seed = std::getenv("PERNIX_TEST_SEED"); if (env_seed == nullptr || *env_seed == '\0') { return defaultSeed; } - char *end = nullptr; + char* end = nullptr; const unsigned long value = std::strtoul(env_seed, &end, 0); return (end != env_seed && *end == '\0') ? static_cast(value) : defaultSeed; } @@ -94,44 +90,42 @@ class TestSet { const T b_max = *std::ranges::max_element(sourceData[i]); const T b_min = *std::ranges::min_element(sourceData[i]); const T b_abs = std::max(std::abs(b_max), std::abs(b_min)); - scalesData[i] = (b_abs > static_cast(0) && quantization_levels > static_cast(0)) - ? (b_abs / quantization_levels) - : std::numeric_limits::epsilon(); + scalesData[i] = (b_abs > static_cast(0) && quantization_levels > static_cast(0)) ? (b_abs / quantization_levels) + : std::numeric_limits::epsilon(); if constexpr (std::is_same_v) { - pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, - sourceData[i].data(), 1.0f / scalesData[i], compressedData[i].data()); + pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, sourceData[i].data(), 1.0f / scalesData[i], + compressedData[i].data()); } else { - pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, - sourceData[i].data(), 1.0 / scalesData[i], compressedData[i].data()); + pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, sourceData[i].data(), 1.0 / scalesData[i], + compressedData[i].data()); } } } }; -template +template struct BitWidthBlockSize { - static constexpr u8 bit_width = BIT_WIDTH; + static constexpr u8 bit_width = BIT_WIDTH; static constexpr u32 block_size = BLOCK_SIZE; }; using testing::Types; -using BitWidthBlockSizeTypes = -Types, BitWidthBlockSize<2, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<3, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<4, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<5, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<6, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<7, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<8, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<9, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<10, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<11, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<12, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<13, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<14, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<15, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<16, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<17, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<18, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<19, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<20, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<21, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<22, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<23, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<24, PERNIX_TEST_BLOCK_SIZE> >; +using BitWidthBlockSizeTypes = Types, BitWidthBlockSize<2, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<3, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<4, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<5, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<6, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<7, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<8, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<9, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<10, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<11, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<12, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<13, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<14, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<15, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<16, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<17, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<18, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<19, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<20, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<21, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<22, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<23, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<24, PERNIX_TEST_BLOCK_SIZE> >; struct BitWidthBlockSizeName { - template + template static std::string GetName(int) { std::ostringstream name; name << "BitWidth" << static_cast(TestConfigT::bit_width) << "BlockSize" << TestConfigT::block_size; @@ -139,90 +133,84 @@ struct BitWidthBlockSizeName { } }; -template +template class CompressionTest : public ::testing::Test { public: - static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u8 BitWidth = TestConfigT::bit_width; static constexpr u32 BlockSize = TestConfigT::block_size; TestSet testSet; - CompressionTest() : testSet(1u << 10) { - } + CompressionTest() : testSet(1u << 10) {} }; -template +template class DecompressionTest : public ::testing::Test { public: - static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u8 BitWidth = TestConfigT::bit_width; static constexpr u32 BlockSize = TestConfigT::block_size; TestSet testSet; - DecompressionTest() : testSet(1u << 10) { - } + DecompressionTest() : testSet(1u << 10) {} }; -template +template class CompressionTest64 : public ::testing::Test { public: - static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u8 BitWidth = TestConfigT::bit_width; static constexpr u32 BlockSize = TestConfigT::block_size; TestSet testSet; - CompressionTest64() : testSet(1u << 10) { - } + CompressionTest64() : testSet(1u << 10) {} }; -template +template class DecompressionTest64 : public ::testing::Test { public: - static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u8 BitWidth = TestConfigT::bit_width; static constexpr u32 BlockSize = TestConfigT::block_size; TestSet testSet; - DecompressionTest64() : testSet(1u << 10) { - } + DecompressionTest64() : testSet(1u << 10) {} }; -template -[[nodiscard]] std::string testContext(const FixtureT &fixture, const u32 block) { +template +[[nodiscard]] std::string testContext(const FixtureT& fixture, const u32 block) { std::ostringstream message; - message << "bit_width=" << static_cast(FixtureT::BitWidth) << ", block_size=" << FixtureT::BlockSize - << ", block=" << block << ", scale=" << fixture.testSet.getScales()[block] - << ", tolerance=" << fixture.testSet.blockTolerance(block) << ", seed=" << fixture.testSet.getSeed(); + message << "bit_width=" << static_cast(FixtureT::BitWidth) << ", block_size=" << FixtureT::BlockSize << ", block=" << block + << ", scale=" << fixture.testSet.getScales()[block] << ", tolerance=" << fixture.testSet.blockTolerance(block) + << ", seed=" << fixture.testSet.getSeed(); return message.str(); } -template -void expectCompressedBlockEqualsReference(const FixtureT &fixture, const std::vector &actual, - const u32 block) { +template +void expectCompressedBlockEqualsReference(const FixtureT& fixture, const std::vector& actual, const u32 block) { SCOPED_TRACE(testContext(fixture, block)); - const auto &expected = fixture.testSet.getCompressedData()[block]; + const auto& expected = fixture.testSet.getCompressedData()[block]; ASSERT_EQ(actual.size(), expected.size()) << "Compressed block byte count differs from reference"; for (u32 byte = 0; byte < actual.size(); byte++) { ASSERT_EQ(actual[byte], expected[byte]) - << "Compressed byte mismatch at byte=" << byte << ", actual=" << static_cast(actual[byte]) - << ", expected=" << static_cast(expected[byte]); + << "Compressed byte mismatch at byte=" << byte << ", actual=" << static_cast(actual[byte]) + << ", expected=" << static_cast(expected[byte]); } } -template -void expectDecompressedBlockNearSource(const FixtureT &fixture, const std::vector &actual, const u32 block) { +template +void expectDecompressedBlockNearSource(const FixtureT& fixture, const std::vector& actual, const u32 block) { SCOPED_TRACE(testContext(fixture, block)); - const auto &expected = fixture.testSet.getDecompressedData()[block]; + const auto& expected = fixture.testSet.getDecompressedData()[block]; ASSERT_EQ(actual.size(), expected.size()) << "Decompressed block element count differs from source"; for (u32 element = 0; element < actual.size(); element++) { ASSERT_NEAR(actual[element], expected[element], fixture.testSet.blockTolerance(block)) - << "Decompressed element mismatch at element=" << element << ", actual=" << actual[element] - << ", expected=" << expected[element] << ", absolute_error=" << std::abs( - actual[element] - expected[element]); + << "Decompressed element mismatch at element=" << element << ", actual=" << actual[element] + << ", expected=" << expected[element] << ", absolute_error=" << std::abs(actual[element] - expected[element]); } } diff --git a/tests/install_consumer/main.c b/tests/install_consumer/main.c index 62c324b..46e739f 100644 --- a/tests/install_consumer/main.c +++ b/tests/install_consumer/main.c @@ -10,12 +10,11 @@ int main(void) { input[i] = (float)((i % 17) - 8) * 0.125f; } - if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 8.0f, - compressed) != PERNIX_STATUS_OK) { + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 8.0f, compressed) != PERNIX_STATUS_OK) { return 1; } - if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 0.125f, - restored, true) != PERNIX_STATUS_OK) { + if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 0.125f, restored, true) != + PERNIX_STATUS_OK) { return 2; } diff --git a/tests/install_consumer/main.cpp b/tests/install_consumer/main.cpp index 7300997..08f70de 100644 --- a/tests/install_consumer/main.cpp +++ b/tests/install_consumer/main.cpp @@ -1,12 +1,11 @@ -#include - #include #include #include +#include int main() { - constexpr u8 bit_width = 8; - constexpr u32 block_size = 64; + constexpr u8 bit_width = 8; + constexpr u32 block_size = 64; constexpr std::size_t elements = (block_size * 8U) / bit_width; std::array input{}; @@ -17,12 +16,10 @@ int main() { std::array compressed{}; std::array restored{}; - if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, 8.0f, compressed) != - PERNIX_STATUS_OK) { + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, 8.0f, compressed) != PERNIX_STATUS_OK) { return 1; } - if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, 0.125f, restored) != - PERNIX_STATUS_OK) { + if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, 0.125f, restored) != PERNIX_STATUS_OK) { return 2; } diff --git a/tests/simd_tests.cpp b/tests/simd_tests.cpp index 019aea3..4ba86e3 100644 --- a/tests/simd_tests.cpp +++ b/tests/simd_tests.cpp @@ -7,21 +7,21 @@ // SIMD compress: compress via backend, decompress via fallback, compare source // --------------------------------------------------------------------------- -template -void testBackendCompressBlock(FixtureT &fixture, pernix_backend backend) { +template +void testBackendCompressBlock(FixtureT& fixture, pernix_backend backend) { using T = std::remove_cvref_t; { std::vector probe(FixtureT::BlockSize); pernix_status st; if constexpr (std::is_same_v) { - st = pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getDecompressedData()[0].data(), - 1.0f / fixture.testSet.getScales()[0], probe.data()); + st = + pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[0].data(), + 1.0f / fixture.testSet.getScales()[0], probe.data()); } else { - st = pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getDecompressedData()[0].data(), - 1.0 / fixture.testSet.getScales()[0], probe.data()); + st = + pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[0].data(), + 1.0 / fixture.testSet.getScales()[0], probe.data()); } if (st != PERNIX_STATUS_OK) { GTEST_SKIP(); @@ -34,23 +34,23 @@ void testBackendCompressBlock(FixtureT &fixture, pernix_backend backend) { pernix_status st; if constexpr (std::is_same_v) { - st = pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getDecompressedData()[b].data(), - 1.0f / fixture.testSet.getScales()[b], compressed.data()); + st = + pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[b].data(), + 1.0f / fixture.testSet.getScales()[b], compressed.data()); } else { - st = pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getDecompressedData()[b].data(), - 1.0 / fixture.testSet.getScales()[b], compressed.data()); + st = + pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[b].data(), + 1.0 / fixture.testSet.getScales()[b], compressed.data()); } ASSERT_EQ(st, PERNIX_STATUS_OK); std::vector restored(fixture.testSet.elementsPerBlock); if constexpr (std::is_same_v) { - st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, - compressed.data(), fixture.testSet.getScales()[b], restored.data(), true); + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, compressed.data(), + fixture.testSet.getScales()[b], restored.data(), true); } else { - st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, - compressed.data(), fixture.testSet.getScales()[b], restored.data(), true); + st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, compressed.data(), + fixture.testSet.getScales()[b], restored.data(), true); } ASSERT_EQ(st, PERNIX_STATUS_OK); @@ -62,21 +62,21 @@ void testBackendCompressBlock(FixtureT &fixture, pernix_backend backend) { // SIMD decompress: decompress fallback-compressed data via backend, compare source // --------------------------------------------------------------------------- -template -void testBackendDecompressBlock(FixtureT &fixture, pernix_backend backend) { +template +void testBackendDecompressBlock(FixtureT& fixture, pernix_backend backend) { using T = std::remove_cvref_t; { std::vector probe(fixture.testSet.elementsPerBlock); pernix_status st; if constexpr (std::is_same_v) { - st = pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getCompressedData()[0].data(), - fixture.testSet.getScales()[0], probe.data(), true); + st = + pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[0].data(), + fixture.testSet.getScales()[0], probe.data(), true); } else { - st = pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getCompressedData()[0].data(), - fixture.testSet.getScales()[0], probe.data(), true); + st = + pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[0].data(), + fixture.testSet.getScales()[0], probe.data(), true); } if (st != PERNIX_STATUS_OK) { GTEST_SKIP(); @@ -89,13 +89,13 @@ void testBackendDecompressBlock(FixtureT &fixture, pernix_backend backend) { pernix_status st; if constexpr (std::is_same_v) { - st = pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getCompressedData()[b].data(), - fixture.testSet.getScales()[b], decompressed.data(), true); + st = + pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed.data(), true); } else { - st = pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, - fixture.testSet.getCompressedData()[b].data(), - fixture.testSet.getScales()[b], decompressed.data(), true); + st = + pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed.data(), true); } ASSERT_EQ(st, PERNIX_STATUS_OK); From 475d8f48a2589601fefb777d9923277b38f5ac1d Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 9 Jul 2026 23:55:49 +0200 Subject: [PATCH 33/43] feat: enhance stdpar fallback backend with improved compression and decompression functions --- CMakeLists.txt | 4 +- README.md | 8 +- include/pernix/fallback/stdpar_common.h | 68 ++--- include/pernix/fallback/stdpar_compression.h | 131 ++++------ .../pernix/fallback/stdpar_decompression.h | 124 ++++----- src/CMakeLists.txt | 25 +- src/fallback/stdpar_compression.cpp | 201 +++++++++++++- src/fallback/stdpar_decompression.cpp | 209 ++++++++++++++- tests/fallback_tests.cpp | 197 +++++++++++++- tests/header_only_tests.cpp | 247 ++++++++++++++++++ 10 files changed, 977 insertions(+), 237 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c88462..7164970 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ option(PERNIX_ENABLE_X86_AVX2 "Build x86 AVX2 backend when targeting x86" ON) option(PERNIX_ENABLE_X86_AVX512VBMI "Build x86 AVX512-VBMI backend when targeting x86" ON) option(PERNIX_ENABLE_ARM64_NEON "Build arm64 NEON backend when targeting arm64" ON) option(PERNIX_ENABLE_ARM64_SVE2 "Build arm64 SVE2 backend when targeting arm64" ON) -set(PERNIX_ENABLE_FALLBACK_STDPAR "OFF" CACHE STRING "Enable the header-only stdpar fallback backend: OFF, AUTO, or ON. AUTO enables it only when TBB is found.") +set(PERNIX_ENABLE_FALLBACK_STDPAR "OFF" CACHE STRING "Enable the stdpar fallback backend: OFF, AUTO, or ON. TBB is optional and enables parallel standard execution policies when found.") set_property(CACHE PERNIX_ENABLE_FALLBACK_STDPAR PROPERTY STRINGS AUTO ON OFF) option(PERNIX_ENABLE_FALLBACK_SIMD "Build experimental fallback SIMD skeleton backend" OFF) @@ -30,7 +30,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") set(PERNIX_BUNDLE_SIMDE_FOR_INSTALL OFF) string(TOUPPER "${PERNIX_ENABLE_FALLBACK_STDPAR}" PERNIX_ENABLE_FALLBACK_STDPAR) -set(PERNIX_ENABLE_FALLBACK_STDPAR "${PERNIX_ENABLE_FALLBACK_STDPAR}" CACHE STRING "Enable the header-only stdpar fallback backend: OFF, AUTO, or ON. AUTO enables it only when TBB is found." FORCE) +set(PERNIX_ENABLE_FALLBACK_STDPAR "${PERNIX_ENABLE_FALLBACK_STDPAR}" CACHE STRING "Enable the stdpar fallback backend: OFF, AUTO, or ON. TBB is optional and enables parallel standard execution policies when found." FORCE) if (NOT PERNIX_ENABLE_FALLBACK_STDPAR MATCHES "^(AUTO|ON|OFF)$") message(FATAL_ERROR "PERNIX_ENABLE_FALLBACK_STDPAR must be AUTO, ON, or OFF") endif () diff --git a/README.md b/README.md index 9930017..27e78e2 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,10 @@ Current backend enum values: * `PERNIX_BACKEND_FALLBACK_SIMD` x86 SIMD kernels are compiled with per-source ISA flags when enabled. Generic dispatch and fallback code are built for the -baseline target. ARM decompression paths exist, but ARM compression is incomplete/stubbed. The stdpar fallback is currently -a header-only target feature, disabled by default, and the compiled library target does not export stdpar dispatch. +baseline target. ARM decompression paths exist, but ARM compression is incomplete/stubbed. The stdpar fallback is disabled +by default. When enabled, both the compiled library and the header-only target export stdpar dispatch. If TBB is available, +stdpar uses standard parallel execution policies; otherwise it preserves the same 8-value grouping logic while executing +sequentially. ## C++ Example @@ -241,7 +243,7 @@ Common options: * `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON`: build Fortran bindings and Fortran example/test * `-DPERNIX_ENABLE_X86_AVX2=OFF`, `-DPERNIX_ENABLE_X86_BMI2=OFF`, `-DPERNIX_ENABLE_X86_AVX512VBMI=OFF`: disable specific x86 backends -* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF|AUTO|ON`: control the header-only stdpar fallback and TBB dependency +* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF|AUTO|ON`: control the stdpar fallback backend. TBB is optional and enables parallel standard execution policies when found Install: diff --git a/include/pernix/fallback/stdpar_common.h b/include/pernix/fallback/stdpar_common.h index 8d7a5bd..c61ed8d 100644 --- a/include/pernix/fallback/stdpar_common.h +++ b/include/pernix/fallback/stdpar_common.h @@ -3,65 +3,21 @@ #include -#include -#include +#include +#include +#if defined(PERNIX_STDPAR_USE_PARALLEL_POLICY) #include -#include +#endif +#include +#include namespace pernix::internal { -template -struct stdpar_buffer { - std::array data{}; - usize size = 0; -}; - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline i32 sign_extend_stdpar(const u32 value) { - if constexpr (BIT_WIDTH == 1) { - return static_cast(value & 1U); - } - - constexpr u32 sign_bit = u32{1} << (BIT_WIDTH - 1); - constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1U; - const u32 masked = value & mask; - return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); -} +inline constexpr u32 stdpar_group_size_v = 8; template requires(GROUP_SIZE >= 1 && GROUP_SIZE <= 8 && BIT_WIDTH >= 1 && BIT_WIDTH <= 24) inline constexpr usize packed_group_bytes_v = (static_cast(GROUP_SIZE) * BIT_WIDTH + 7U) / 8U; -template - requires(std::is_floating_point_v) -__always_inline ScaleType dequantize_stdpar_value(const i32 input, const ScaleType scale) { - if constexpr (std::is_same_v) { - return static_cast(input) * scale; - } else { - return static_cast(input) * scale; - } -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -__always_inline u32 quantize_stdpar_value(const ScaleType input, const ScaleType scale) { - constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); - constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); - - const long double scaled = static_cast(input) * static_cast(scale); - if (std::isnan(scaled)) { - return 0; - } - if (scaled <= static_cast(min_value)) { - return static_cast(static_cast(min_value)); - } - if (scaled >= static_cast(max_value)) { - return static_cast(static_cast(max_value)); - } - - return static_cast(static_cast(std::llround(scaled))); -} - template class counting_iterator { public: @@ -144,6 +100,16 @@ class counting_iterator { private: T value_; }; + +template +__always_inline void for_each_index_stdpar(const u32 count, Func&& func) { + auto first = counting_iterator{0}; +#if defined(PERNIX_STDPAR_USE_PARALLEL_POLICY) + std::for_each_n(std::execution::par_unseq, first, count, std::forward(func)); +#else + std::for_each_n(first, count, std::forward(func)); +#endif +} } // namespace pernix::internal #endif // PERNIX_FALLBACK_STDPAR_COMMON_H diff --git a/include/pernix/fallback/stdpar_compression.h b/include/pernix/fallback/stdpar_compression.h index f8aab9e..f6b8283 100644 --- a/include/pernix/fallback/stdpar_compression.h +++ b/include/pernix/fallback/stdpar_compression.h @@ -1,112 +1,81 @@ #ifndef PERNIX_FALLBACK_STDPAR_COMPRESSION_H #define PERNIX_FALLBACK_STDPAR_COMPRESSION_H +#include +#include #include -#include -#include -#include +#include +#include #include #include namespace pernix { -namespace internal { -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && GROUP_SIZE >= 1 && GROUP_SIZE <= 8 && std::is_floating_point_v) -__always_inline stdpar_buffer quantize_epi32_group_fallback_stdpar(const std::span input, - const ScaleType scale) { - stdpar_buffer output; - -#pragma GCC unroll GROUP_SIZE - for (usize i = 0; i < GROUP_SIZE; ++i) { - output.data[i] = quantize_stdpar_value(input[i], scale); - ++output.size; - } - - return output; -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && GROUP_SIZE >= 1 && GROUP_SIZE <= 8) -__always_inline void pack_epi32_group_fallback_stdpar(const stdpar_buffer& input, const std::span destination) { - constexpr u32 bitmask = (u32{1} << BIT_WIDTH) - 1U; - - usize idx = 0; - usize bits_in_buffer = 0; - u64 buffer = 0; - -#pragma GCC unroll GROUP_SIZE - for (usize i = 0; i < GROUP_SIZE; ++i) { - buffer |= (static_cast(input.data[i] & bitmask) << bits_in_buffer); - bits_in_buffer += BIT_WIDTH; - - while (bits_in_buffer >= 8) { - destination[idx++] = static_cast(buffer & 0xFFU); - buffer >>= 8; - bits_in_buffer -= 8; - } - } - - if (bits_in_buffer > 0) { - destination[idx] = static_cast(buffer & 0xFFU); - } -} -} // namespace internal - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int compress_block_fallback_stdpar(const void* input_ptr, ScaleType scale, void* output_ptr) { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_block_fallback_stdpar(const std::span input, const FloatT scale, std::span destination) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; - constexpr u32 full_groups = elements_per_block / 8U; - constexpr u32 remainder = elements_per_block % 8U; + constexpr u32 full_groups = elements_per_block / internal::stdpar_group_size_v; + constexpr u32 remainder = elements_per_block % internal::stdpar_group_size_v; - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + std::ranges::fill(destination, 0); - std::memset(output, 0, BLOCK_SIZE); + internal::for_each_index_stdpar(full_groups, [&](const u32 group) { + constexpr u32 group_size = internal::stdpar_group_size_v; - auto first = internal::counting_iterator{0}; - std::for_each_n(std::execution::par_unseq, first, full_groups, [&](const u32 group) { - constexpr u8 group_size = 8; - constexpr usize packed_group_bytes = internal::packed_group_bytes_v; + const auto input_group = input.subspan(static_cast(group) * group_size, group_size); + auto destination_group = destination.subspan(static_cast(group) * BIT_WIDTH, BIT_WIDTH); - const auto quantized = internal::quantize_epi32_group_fallback_stdpar( - std::span(input + static_cast(group) * group_size, group_size), scale); - internal::pack_epi32_group_fallback_stdpar( - quantized, std::span(output + static_cast(group) * BIT_WIDTH, packed_group_bytes)); + internal::quantize_and_pack_fallback(input_group, scale, group_size, destination_group); }); if constexpr (remainder != 0) { - constexpr u8 tail_group_size = static_cast(remainder); - constexpr usize tail_input_offset = full_groups * 8U; + constexpr usize tail_input_offset = full_groups * internal::stdpar_group_size_v; constexpr usize tail_output_offset = full_groups * BIT_WIDTH; - constexpr usize tail_bytes = internal::packed_group_bytes_v; + constexpr usize tail_bytes = internal::packed_group_bytes_v(remainder), BIT_WIDTH>; - const auto quantized = internal::quantize_epi32_group_fallback_stdpar( - std::span(input + tail_input_offset, tail_group_size), scale); - internal::pack_epi32_group_fallback_stdpar(quantized, - std::span(output + tail_output_offset, tail_bytes)); + internal::quantize_and_pack_fallback(input.subspan(tail_input_offset, remainder), scale, remainder, + destination.subspan(tail_output_offset, tail_bytes)); } - return PERNIX_STATUS_OK; + return 0; } -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int compress_blocks_fallback_stdpar(const void* input_ptr, ScaleType scale, void* output_ptr, u32 blocks) { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_block_fallback_stdpar(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr) { constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); + const auto input_span = std::span(static_cast(input_ptr), elements_per_block); + auto output_span = std::span(static_cast(output_ptr), BLOCK_SIZE); - auto first = internal::counting_iterator{0}; - std::for_each_n(std::execution::par_unseq, first, blocks, [&](const u32 block) { - const auto* block_input = input + (static_cast(block) * elements_per_block); - auto* block_output = output + (static_cast(block) * BLOCK_SIZE); - static_cast(compress_block_fallback_stdpar(block_input, scale, block_output)); + return compress_block_fallback_stdpar(input_span, scale, output_span); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_blocks_fallback_stdpar(const std::span input, const FloatT scale, std::span destination, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + internal::for_each_index_stdpar(blocks, [&](const u32 block) { + const auto block_input = input.subspan(static_cast(block) * elements_per_block, elements_per_block); + auto block_output = destination.subspan(static_cast(block) * BLOCK_SIZE, BLOCK_SIZE); + static_cast(compress_block_fallback_stdpar(block_input, scale, block_output)); }); - return PERNIX_STATUS_OK; + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_blocks_fallback_stdpar(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr, + const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), elements_per_block * blocks); + auto output_span = std::span(static_cast(output_ptr), static_cast(blocks) * BLOCK_SIZE); + + return compress_blocks_fallback_stdpar(input_span, scale, output_span, blocks); } } // namespace pernix diff --git a/include/pernix/fallback/stdpar_decompression.h b/include/pernix/fallback/stdpar_decompression.h index da85797..1b18080 100644 --- a/include/pernix/fallback/stdpar_decompression.h +++ b/include/pernix/fallback/stdpar_decompression.h @@ -1,109 +1,79 @@ #ifndef PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H #define PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H +#include +#include #include -#include +#include #include #include -#include "scalar_decompression.h" - namespace pernix { -namespace internal { -// Eight packed values consume exactly BIT_WIDTH bytes. -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && GROUP_SIZE >= 1 && GROUP_SIZE <= 8) -__always_inline stdpar_buffer unpack_epi32_group_fallback_stdpar(const std::span input) { - constexpr auto bit_offset = 0; // TODO: for now - constexpr u32 bitmask = (u32{1} << BIT_WIDTH) - 1U; - - stdpar_buffer output; - - // Unpack one fixed-size byte group using the scalar bit-buffer logic. - - usize idx = 0; - u8 bits_in_buffer = 8 - bit_offset; - u64 buffer = static_cast(input[idx++]) >> bit_offset; - -#pragma GCC unroll GROUP_SIZE - for (usize i = 0; i < GROUP_SIZE; i++) { - while (BIT_WIDTH > bits_in_buffer) { - const auto next_value = static_cast(input[idx++]) << bits_in_buffer; - buffer |= next_value; - bits_in_buffer += 8; - } - - const u32 raw_value = static_cast(buffer & bitmask); - if constexpr (SIGN_VALUES) { - output.data[i] = sign_extend_stdpar(raw_value); - } else { - output.data[i] = static_cast(raw_value); - } - ++output.size; - - buffer >>= BIT_WIDTH; - bits_in_buffer -= BIT_WIDTH; - } - return output; -} -} // namespace internal - -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int decompress_block_fallback_stdpar(const void* __restrict__ input_ptr, const ScaleType scale, void* __restrict__ output_ptr) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr u32 full_groups = elements_per_block / 8; - constexpr u32 remainder = elements_per_block % 8; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_block_fallback_stdpar(const std::span input, const FloatT scale, std::span output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + constexpr u32 full_groups = elements_per_block / internal::stdpar_group_size_v; + constexpr u32 remainder = elements_per_block % internal::stdpar_group_size_v; - const auto input_span = std::span(static_cast(input_ptr), BLOCK_SIZE); - auto output_span = std::span(static_cast(output_ptr), elements_per_block); + internal::for_each_index_stdpar(full_groups, [&](const u32 group) { + constexpr u32 group_size = internal::stdpar_group_size_v; - auto first = internal::counting_iterator{0}; - std::for_each_n(std::execution::par_unseq, first, full_groups, [&](const u32 group) { - const auto input_subspan = input_span.subspan(static_cast(group) * BIT_WIDTH, BIT_WIDTH); - const auto unpacked = internal::unpack_epi32_group_fallback_stdpar(input_subspan); + const auto input_group = input.subspan(static_cast(group) * BIT_WIDTH, BIT_WIDTH); + auto output_group = output.subspan(static_cast(group) * group_size, group_size); - for (u32 i = 0; i < unpacked.size; ++i) { - output_span[static_cast(group) * 8U + i] = internal::dequantize_stdpar_value(unpacked.data[i], scale); - } + internal::unpack_and_dequantize_fallback(input_group, group_size, scale, output_group); }); if constexpr (remainder != 0) { constexpr usize tail_input_offset = full_groups * BIT_WIDTH; - constexpr usize tail_output_offset = full_groups * 8U; + constexpr usize tail_output_offset = full_groups * internal::stdpar_group_size_v; constexpr usize tail_bytes = internal::packed_group_bytes_v(remainder), BIT_WIDTH>; - const auto unpacked = internal::unpack_epi32_group_fallback_stdpar(remainder)>( - input_span.subspan(tail_input_offset, tail_bytes)); - - for (u32 i = 0; i < remainder; ++i) { - output_span[tail_output_offset + i] = internal::dequantize_stdpar_value(unpacked.data[i], scale); - } + internal::unpack_and_dequantize_fallback(input.subspan(tail_input_offset, tail_bytes), remainder, scale, + output.subspan(tail_output_offset, remainder)); } return 0; } -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v) -int decompress_blocks_fallback_stdpar(const void* __restrict__ input_ptr, const ScaleType scale, void* __restrict__ output_ptr, - const u32 blocks) { - constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_block_fallback_stdpar(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), BLOCK_SIZE); + auto output_span = std::span(static_cast(output_ptr), elements_per_block); + + return decompress_block_fallback_stdpar(input_span, scale, output_span); +} - const auto* input = static_cast(input_ptr); - auto* output = static_cast(output_ptr); +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_blocks_fallback_stdpar(const std::span input, const FloatT scale, std::span output, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; - auto first = internal::counting_iterator{0}; - std::for_each_n(std::execution::par_unseq, first, blocks, [&](const u32 block) { - const auto* block_input = input + (static_cast(block) * BLOCK_SIZE); - auto* block_output = output + (static_cast(block) * elements_per_block); - static_cast( - decompress_block_fallback_stdpar(block_input, scale, block_output)); + internal::for_each_index_stdpar(blocks, [&](const u32 block) { + const auto block_input = input.subspan(static_cast(block) * BLOCK_SIZE, BLOCK_SIZE); + auto block_output = output.subspan(static_cast(block) * elements_per_block, elements_per_block); + static_cast(decompress_block_fallback_stdpar(block_input, scale, block_output)); }); return 0; } + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_blocks_fallback_stdpar(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr, + const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), static_cast(blocks) * BLOCK_SIZE); + auto output_span = std::span(static_cast(output_ptr), elements_per_block * blocks); + + return decompress_blocks_fallback_stdpar(input_span, scale, output_span, blocks); +} } // namespace pernix #endif // PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index be7dd5d..dc18d89 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -72,19 +72,24 @@ endif () set(PERNIX_BUILD_FALLBACK_STDPAR OFF) if (NOT PERNIX_ENABLE_FALLBACK_STDPAR STREQUAL "OFF") - if (PERNIX_ENABLE_FALLBACK_STDPAR STREQUAL "ON") - find_package(TBB REQUIRED) - else () - find_package(TBB QUIET) - endif () + set(PERNIX_BUILD_FALLBACK_STDPAR ON) + find_package(TBB QUIET) + + target_sources(pernix PRIVATE + fallback/stdpar_compression.cpp + fallback/stdpar_decompression.cpp + ) + target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_STDPAR=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_STDPAR=1) if (TARGET TBB::tbb) - set(PERNIX_BUILD_FALLBACK_STDPAR ON) - target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_STDPAR=1) + target_compile_definitions(pernix PUBLIC PERNIX_STDPAR_USE_PARALLEL_POLICY=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_STDPAR_USE_PARALLEL_POLICY=1) + target_link_libraries(pernix PUBLIC TBB::tbb) target_link_libraries(pernix_header_only INTERFACE TBB::tbb) - message(STATUS "PERNIX header-only fallback stdpar backend enabled (TBB found)") - elseif (PERNIX_ENABLE_FALLBACK_STDPAR STREQUAL "AUTO") - message(STATUS "PERNIX header-only fallback stdpar backend disabled (TBB not found; set PERNIX_ENABLE_FALLBACK_STDPAR=ON to require it)") + message(STATUS "PERNIX fallback stdpar backend enabled with parallel policy (TBB found)") + else () + message(STATUS "PERNIX fallback stdpar backend enabled without TBB; stdpar will execute sequentially") endif () endif () diff --git a/src/fallback/stdpar_compression.cpp b/src/fallback/stdpar_compression.cpp index 3db39cc..7d982de 100644 --- a/src/fallback/stdpar_compression.cpp +++ b/src/fallback/stdpar_compression.cpp @@ -1,4 +1,201 @@ #include +#include -// Header-only selector definitions live in select_impl.h. -// This translation unit reserves the compiled stdpar compression backend slot. +namespace pernix::internal { +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +Kernel select_fallback_stdpar_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal diff --git a/src/fallback/stdpar_decompression.cpp b/src/fallback/stdpar_decompression.cpp index d0768d7..9bd51ce 100644 --- a/src/fallback/stdpar_decompression.cpp +++ b/src/fallback/stdpar_decompression.cpp @@ -1,4 +1,209 @@ #include +#include -// Header-only selector definitions live in select_impl.h. -// This translation unit reserves the compiled stdpar decompression backend slot. +namespace pernix::internal { +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +Kernel select_fallback_stdpar_decompress_block_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_decompress_block_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp index 587e12e..f7629af 100644 --- a/tests/fallback_tests.cpp +++ b/tests/fallback_tests.cpp @@ -51,6 +51,65 @@ void expect_backend_decompressed_block_matches_source(FixtureT& fixture, const p expectDecompressedBlockNearSource(fixture, decompressed[b], b); } } + +template +void expect_backend_compress_blocks_matches_scalar(FixtureT& fixture, CompressFn compress_fn, const pernix_backend backend) { + using ScaleType = std::remove_cvref_t; + + const u32 blocks = fixture.testSet.numberOfBlocks; + const u32 elements_per_block = fixture.testSet.elementsPerBlock; + const u32 total_elements = blocks * elements_per_block; + const usize compressed_bytes = static_cast(blocks) * FixtureT::BlockSize; + std::vector input(total_elements); + + for (u32 block = 0; block < blocks; ++block) { + std::copy_n(fixture.testSet.getDecompressedData()[block].data(), elements_per_block, + input.data() + static_cast(block) * elements_per_block); + } + + const ScaleType scale_inv = ScaleType{1} / fixture.testSet.getScales()[0]; + std::vector scalar_output(compressed_bytes); + std::vector stdpar_output(compressed_bytes); + + auto status = compress_fn(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, input.data(), scale_inv, scalar_output.data(), + blocks); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + status = compress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, input.data(), scale_inv, stdpar_output.data(), blocks); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_output, scalar_output); +} + +template +void expect_backend_decompress_blocks_matches_scalar(FixtureT& fixture, DecompressFn decompress_fn, const pernix_backend backend, + const bool sign_values = true) { + using ValueType = std::remove_cvref_t; + + const u32 blocks = fixture.testSet.numberOfBlocks; + const u32 elements_per_block = fixture.testSet.elementsPerBlock; + const u32 total_elements = blocks * elements_per_block; + std::vector compressed(static_cast(blocks) * FixtureT::BlockSize); + + for (u32 block = 0; block < blocks; ++block) { + std::copy_n(fixture.testSet.getCompressedData()[block].data(), FixtureT::BlockSize, + compressed.data() + static_cast(block) * FixtureT::BlockSize); + } + + std::vector scalar_output(total_elements); + std::vector stdpar_output(total_elements); + const auto scale = fixture.testSet.getScales()[0]; + + auto status = decompress_fn(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, compressed.data(), scale, + scalar_output.data(), blocks, sign_values); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + status = decompress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, compressed.data(), scale, stdpar_output.data(), blocks, + sign_values); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_output, scalar_output); +} } // namespace // --------------------------------------------------------------------------- @@ -145,6 +204,22 @@ TYPED_TEST(DecompressionTest, FallbackStdparDecompressBlock) { TYPED_TEST(DecompressionTest64, FallbackStdparDecompressBlock) { expect_backend_decompressed_block_matches_source(*this, PERNIX_BACKEND_FALLBACK_STDPAR, &pernix_decompress_block_f64); } + +TYPED_TEST(CompressionTest, FallbackStdparCompressBlocksMatchesScalar) { + expect_backend_compress_blocks_matches_scalar(*this, &pernix_compress_blocks_f32, PERNIX_BACKEND_FALLBACK_STDPAR); +} + +TYPED_TEST(CompressionTest64, FallbackStdparCompressBlocksMatchesScalar) { + expect_backend_compress_blocks_matches_scalar(*this, &pernix_compress_blocks_f64, PERNIX_BACKEND_FALLBACK_STDPAR); +} + +TYPED_TEST(DecompressionTest, FallbackStdparDecompressBlocksMatchesScalar) { + expect_backend_decompress_blocks_matches_scalar(*this, &pernix_decompress_blocks_f32, PERNIX_BACKEND_FALLBACK_STDPAR); +} + +TYPED_TEST(DecompressionTest64, FallbackStdparDecompressBlocksMatchesScalar) { + expect_backend_decompress_blocks_matches_scalar(*this, &pernix_decompress_blocks_f64, PERNIX_BACKEND_FALLBACK_STDPAR); +} #endif // --------------------------------------------------------------------------- @@ -546,9 +621,14 @@ TEST(FallbackStdparEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { constexpr u32 BS = 64; const std::array input{0x08}; + std::array scalar_output{}; std::array output{}; - const auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, 4, BS, input.data(), 1.0f, output.data(), true); + auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, scalar_output.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, 4, BS, input.data(), 1.0f, output.data(), true); ASSERT_EQ(st, PERNIX_STATUS_OK); + EXPECT_EQ(output, scalar_output); EXPECT_EQ(output[0], -8.0f); } @@ -576,18 +656,117 @@ TEST(FallbackStdparEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { input[1] = -std::numeric_limits::infinity(); input[2] = std::numeric_limits::quiet_NaN(); - std::array compressed{}; - std::array restored{}; + std::array scalar_compressed{}; + std::array stdpar_compressed{}; + std::array scalar_restored{}; + std::array stdpar_restored{}; - auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, compressed.data()); + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, scalar_compressed.data()); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, restored.data(), true); + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, stdpar_compressed.data()); ASSERT_EQ(st, PERNIX_STATUS_OK); - EXPECT_EQ(restored[0], 7.0f); - EXPECT_EQ(restored[1], -8.0f); - EXPECT_EQ(restored[2], 0.0f); + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, scalar_compressed.data(), 1.0f, scalar_restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, stdpar_compressed.data(), 1.0f, stdpar_restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_compressed, scalar_compressed); + EXPECT_EQ(stdpar_restored, scalar_restored); + EXPECT_EQ(stdpar_restored[0], 7.0f); + EXPECT_EQ(stdpar_restored[1], -8.0f); + EXPECT_EQ(stdpar_restored[2], 0.0f); +} + +TEST(FallbackStdparEdgeTest, SignValuesFalseMatchesScalar) { + constexpr u32 BS = 64; + const std::array input{0x0F}; + + std::array scalar_output{}; + std::array stdpar_output{}; + + auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, scalar_output.data(), false); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, 4, BS, input.data(), 1.0f, stdpar_output.data(), false); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_output, scalar_output); + EXPECT_EQ(stdpar_output[0], 15.0f); +} + +TEST(FallbackStdparEdgeTest, RoundingAndClampingMatchScalar) { + constexpr u32 BS = 64; + constexpr u32 BW = 4; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + input[0] = 0.49f; + input[1] = 0.50f; + input[2] = 1.49f; + input[3] = 1.50f; + input[4] = 7.49f; + input[5] = 7.50f; + input[6] = -7.49f; + input[7] = -7.50f; + input[8] = 99.0f; + input[9] = -99.0f; + + std::array scalar_compressed{}; + std::array stdpar_compressed{}; + std::array scalar_restored{}; + std::array stdpar_restored{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, scalar_compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, stdpar_compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, scalar_compressed.data(), 1.0f, scalar_restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, stdpar_compressed.data(), 1.0f, stdpar_restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_compressed, scalar_compressed); + EXPECT_EQ(stdpar_restored, scalar_restored); +} + +TEST(FallbackStdparEdgeTest, RepeatedExecutionIsDeterministic) { + constexpr u32 BS = 64; + constexpr u32 BW = 12; + constexpr u32 blocks = 3; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < input.size(); ++i) { + input[i] = static_cast((static_cast(i % 23U) - 11) * 0.25f); + } + + std::array first_compressed{}; + std::array second_compressed{}; + std::array first_restored{}; + std::array second_restored{}; + + auto st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 4.0f, first_compressed.data(), blocks); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 4.0f, second_compressed.data(), blocks); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, first_compressed.data(), 0.25f, first_restored.data(), blocks, + true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, second_compressed.data(), 0.25f, + second_restored.data(), blocks, true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(second_compressed, first_compressed); + EXPECT_EQ(second_restored, first_restored); } TEST(FallbackStdparEdgeTest, TailWidthCompressMatchesScalarF32) { @@ -764,7 +943,7 @@ TEST(ErrorCodeTest, FallbackAliasMatchesScalar) { EXPECT_EQ(PERNIX_BACKEND_FALLBACK, PERNIX_BACKEND_FALLBACK_SCALAR); } -TEST(ErrorCodeTest, FallbackStdparReturnsUnsupportedImplementationByDefault) { +TEST(ErrorCodeTest, FallbackStdparAvailabilityMatchesBuildConfiguration) { constexpr u32 BS = 64; constexpr u8 BW = 8; diff --git a/tests/header_only_tests.cpp b/tests/header_only_tests.cpp index 52ac3e1..ea0ac12 100644 --- a/tests/header_only_tests.cpp +++ b/tests/header_only_tests.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -36,6 +37,125 @@ void expect_round_trip(const pernix::Backend backend) { EXPECT_NEAR(restored[i], input[i], static_cast(1.0)); } } + +template +std::vector make_block_pattern(const u8 bit_width, const u32 blocks) { + const u32 elements_per_block = pernix::elements_per_block(bit_width); + std::vector values(static_cast(elements_per_block) * blocks); + + for (usize i = 0; i < values.size(); ++i) { + values[i] = static_cast((static_cast(i % 29U) - 14) * 0.25); + } + + return values; +} + +template +void expect_stdpar_matches_scalar_for_all_bit_widths() { + constexpr u32 blocks = 3; + + for (u8 bit_width = pernix::min_bit_width(); bit_width <= pernix::max_bit_width(); ++bit_width) { + const u32 elements_per_block = pernix::elements_per_block(bit_width); + auto input = make_block_pattern(bit_width, blocks); + + std::vector scalar_block(kBlockSize); + std::vector stdpar_block(kBlockSize); + std::vector scalar_blocks(static_cast(kBlockSize) * blocks); + std::vector stdpar_blocks(static_cast(kBlockSize) * blocks); + std::vector scalar_restored(elements_per_block * blocks); + std::vector stdpar_restored(elements_per_block * blocks); + + ASSERT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, + std::span(input.data(), elements_per_block), static_cast(4), + std::span(scalar_block)), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + ASSERT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, + std::span(input.data(), elements_per_block), static_cast(4), + std::span(stdpar_block)), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + EXPECT_EQ(stdpar_block, scalar_block) << "bit_width=" << static_cast(bit_width); + + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), + static_cast(4), std::span(scalar_blocks), blocks), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), + static_cast(4), std::span(stdpar_blocks), blocks), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + EXPECT_EQ(stdpar_blocks, scalar_blocks) << "bit_width=" << static_cast(bit_width); + + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(scalar_blocks), + static_cast(0.25), std::span(scalar_restored), blocks, true), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(stdpar_blocks), + static_cast(0.25), std::span(stdpar_restored), blocks, true), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + EXPECT_EQ(stdpar_restored, scalar_restored) << "bit_width=" << static_cast(bit_width); + } +} + +template +void expect_cross_compatibility(const u8 bit_width, const u32 blocks, const FloatT compression_scale, const FloatT decompression_scale, + const bool sign_values = true) { + const u32 elements_per_block = pernix::elements_per_block(bit_width); + auto input = make_block_pattern(bit_width, blocks); + + std::vector scalar_compressed(static_cast(kBlockSize) * blocks); + std::vector stdpar_compressed(static_cast(kBlockSize) * blocks); + std::vector scalar_to_stdpar(static_cast(elements_per_block) * blocks, static_cast(0)); + std::vector stdpar_to_scalar(static_cast(elements_per_block) * blocks, static_cast(0)); + + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), compression_scale, + std::span(scalar_compressed), blocks), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), + compression_scale, std::span(stdpar_compressed), blocks), + PERNIX_STATUS_OK); + + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(scalar_compressed), + decompression_scale, std::span(scalar_to_stdpar), blocks, sign_values), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(stdpar_compressed), + decompression_scale, std::span(stdpar_to_scalar), blocks, sign_values), + PERNIX_STATUS_OK); + + EXPECT_EQ(scalar_to_stdpar, stdpar_to_scalar) << "bit_width=" << static_cast(bit_width); +} + +template +void expect_single_block_partial_group_matches_scalar(const u8 bit_width) { + const u32 elements_per_block = pernix::elements_per_block(bit_width); + auto input = make_block_pattern(bit_width, 1); + + ASSERT_GT(elements_per_block, 8U) << "bit_width=" << static_cast(bit_width); + ASSERT_NE(elements_per_block % 8U, 0U) << "bit_width=" << static_cast(bit_width); + + std::vector scalar_compressed(kBlockSize); + std::vector stdpar_compressed(kBlockSize); + std::vector scalar_restored(elements_per_block, static_cast(0)); + std::vector stdpar_restored(elements_per_block, static_cast(0)); + + ASSERT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), static_cast(4), + std::span(scalar_compressed)), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), + static_cast(4), std::span(stdpar_compressed)), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_compressed, scalar_compressed) << "bit_width=" << static_cast(bit_width); + + ASSERT_EQ(pernix::decompress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(scalar_compressed), + static_cast(0.25), std::span(scalar_restored), true), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(stdpar_compressed), + static_cast(0.25), std::span(stdpar_restored), true), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_restored, scalar_restored) << "bit_width=" << static_cast(bit_width); +} } // namespace TEST(HeaderOnlyPernix, FallbackFloatRoundTrip) { @@ -168,6 +288,133 @@ TEST(HeaderOnlyPernix, FallbackStdparReturnsUnsupportedImplementation) { #endif } +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TEST(HeaderOnlyPernix, FallbackStdparMatchesScalarForAllBitWidthsF32) { + expect_stdpar_matches_scalar_for_all_bit_widths(); +} + +TEST(HeaderOnlyPernix, FallbackStdparMatchesScalarForAllBitWidthsF64) { + expect_stdpar_matches_scalar_for_all_bit_widths(); +} + +TEST(HeaderOnlyPernix, FallbackStdparMatchesScalarForEdgeValues) { + constexpr u8 bit_width = 4; + constexpr u32 blocks = 2; + + const u32 elements_per_block = pernix::elements_per_block(bit_width); + std::vector input(static_cast(elements_per_block) * blocks, 0.0f); + input[0] = 0.49f; + input[1] = 0.50f; + input[2] = 1.49f; + input[3] = 1.50f; + input[4] = std::numeric_limits::infinity(); + input[5] = -std::numeric_limits::infinity(); + input[6] = std::numeric_limits::quiet_NaN(); + input[7] = 99.0f; + input[8] = -99.0f; + + std::vector scalar_compressed(static_cast(kBlockSize) * blocks); + std::vector stdpar_compressed(static_cast(kBlockSize) * blocks); + std::vector scalar_signed(input.size()); + std::vector stdpar_signed(input.size()); + + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), 1.0f, + std::span(scalar_compressed), blocks), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 1.0f, + std::span(stdpar_compressed), blocks), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_compressed, scalar_compressed); + + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(scalar_compressed), 1.0f, + std::span(scalar_signed), blocks, true), + PERNIX_STATUS_OK); + ASSERT_EQ( + pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(stdpar_compressed), 1.0f, + std::span(stdpar_signed), blocks, true), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_signed, scalar_signed); +} + +TEST(HeaderOnlyPernix, FallbackStdparSingleBlockPartialGroupsMatchScalar) { + for (const u8 bit_width : std::array{3, 5, 6, 7, 10, 11, 12, 24}) { + expect_single_block_partial_group_matches_scalar(bit_width); + } +} + +TEST(HeaderOnlyPernix, FallbackStdparCrossCompatibilityMatchesScalarForSignedValues) { + for (const u8 bit_width : std::array{1, 3, 5, 8, 9, 12, 17, 24}) { + expect_cross_compatibility(bit_width, 2, 4.0f, 0.25f, true); + } +} + +TEST(HeaderOnlyPernix, FallbackStdparCrossCompatibilityMatchesScalarForUnsignedDecode) { + expect_cross_compatibility(4, 2, 1.0f, 1.0f, false); +} + +TEST(HeaderOnlyPernix, FallbackStdparSignValuesFalseMatchesScalar) { + constexpr u8 bit_width = 4; + const u32 elements_per_block = pernix::elements_per_block(bit_width); + + std::vector compressed(kBlockSize, 0); + compressed[0] = 0x0F; + + std::vector scalar_output(elements_per_block, 0.0f); + std::vector stdpar_output(elements_per_block, 0.0f); + + ASSERT_EQ(pernix::decompress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(compressed), 1.0f, + std::span(scalar_output), false), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(compressed), 1.0f, + std::span(stdpar_output), false), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_output, scalar_output); +} + +TEST(HeaderOnlyPernix, FallbackStdparRepeatedExecutionIsDeterministic) { + constexpr u8 bit_width = 12; + constexpr u32 blocks = 3; + + auto input = make_block_pattern(bit_width, blocks); + std::vector first_compressed(static_cast(kBlockSize) * blocks); + std::vector second_compressed(static_cast(kBlockSize) * blocks); + std::vector first_restored(input.size(), 0.0f); + std::vector second_restored(input.size(), 0.0f); + + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 4.0f, + std::span(first_compressed), blocks), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 4.0f, + std::span(second_compressed), blocks), + PERNIX_STATUS_OK); + EXPECT_EQ(second_compressed, first_compressed); + + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(first_compressed), + 0.25f, std::span(first_restored), blocks, true), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(second_compressed), + 0.25f, std::span(second_restored), blocks, true), + PERNIX_STATUS_OK); + EXPECT_EQ(second_restored, first_restored); +} + +TEST(HeaderOnlyPernix, FallbackStdparRepeatedSingleBlockCompressionIsDeterministic) { + constexpr u8 bit_width = 17; + + auto input = make_block_pattern(bit_width, 1); + std::vector first_compressed(kBlockSize); + std::vector second_compressed(kBlockSize); + + ASSERT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 2.0, + std::span(first_compressed)), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 2.0, + std::span(second_compressed)), + PERNIX_STATUS_OK); + EXPECT_EQ(second_compressed, first_compressed); +} +#endif + TEST(HeaderOnlyPernix, FallbackSimdReturnsUnsupportedImplementation) { auto input = make_input(); std::vector compressed(kBlockSize); From a4f0a5d9d5d43038a4b9c8f3ec44f296f057319e Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Fri, 10 Jul 2026 00:15:36 +0200 Subject: [PATCH 34/43] feat: rename tolerance function and enhance tolerance calculation for improved accuracy --- tests/deterministic_correctness_tests.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/deterministic_correctness_tests.cpp b/tests/deterministic_correctness_tests.cpp index 692babe..8345a8c 100644 --- a/tests/deterministic_correctness_tests.cpp +++ b/tests/deterministic_correctness_tests.cpp @@ -66,8 +66,13 @@ pernix_status compute_scale(const FloatT bmax, const u8 bit_width, FloatT& scale } template -FloatT tolerance_for_scale(const FloatT scale) { - return (abs_value(scale) * static_cast(0.55)) + (std::numeric_limits::epsilon() * static_cast(64)); +FloatT tolerance_for_value(const FloatT input, const FloatT scale) { + // Quantization contributes half a step in exact arithmetic. Reciprocal scaling and + // dequantization add magnitude-dependent rounding error, which is visible for f32 + // inputs whose magnitude approaches the number of exactly representable integers. + const FloatT epsilon = std::numeric_limits::epsilon(); + return (abs_value(scale) * static_cast(0.55)) + + (epsilon * ((abs_value(input) * static_cast(2)) + static_cast(64))); } template @@ -134,8 +139,8 @@ std::vector make_dataset(const Pattern pattern, const u8 bit_width, cons template void expect_near_input(const std::vector& input, const std::vector& restored, const FloatT scale) { ASSERT_EQ(restored.size(), input.size()); - const FloatT tolerance = tolerance_for_scale(scale); for (usize i = 0; i < input.size(); ++i) { + const FloatT tolerance = tolerance_for_value(input[i], scale); ASSERT_NEAR(restored[i], input[i], tolerance) << "element=" << i << ", scale=" << scale; } } From d62a5cdba368fa1ad2bd87d594bd3531f635cc46 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Fri, 10 Jul 2026 22:01:41 +0200 Subject: [PATCH 35/43] feat: update CMake configuration to enable experimental C++26 std::simd fallback backend --- CMakeLists.txt | 58 ++++++++++++++++++++++++++++++++++++++++++++-- README.md | 3 +++ src/CMakeLists.txt | 8 ++++++- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7164970..da59443 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.24) +cmake_minimum_required(VERSION 3.25) project(pernix VERSION 0.1.0 LANGUAGES CXX C) option(BUILD_SHARED_LIBS "Build pernix as a shared library by default" ON) @@ -20,7 +20,8 @@ option(PERNIX_ENABLE_ARM64_NEON "Build arm64 NEON backend when targeting arm64" option(PERNIX_ENABLE_ARM64_SVE2 "Build arm64 SVE2 backend when targeting arm64" ON) set(PERNIX_ENABLE_FALLBACK_STDPAR "OFF" CACHE STRING "Enable the stdpar fallback backend: OFF, AUTO, or ON. TBB is optional and enables parallel standard execution policies when found.") set_property(CACHE PERNIX_ENABLE_FALLBACK_STDPAR PROPERTY STRINGS AUTO ON OFF) -option(PERNIX_ENABLE_FALLBACK_SIMD "Build experimental fallback SIMD skeleton backend" OFF) +set(PERNIX_ENABLE_FALLBACK_SIMD "AUTO" CACHE STRING "Enable the C++26 std::simd fallback backend: OFF, AUTO, or ON.") +set_property(CACHE PERNIX_ENABLE_FALLBACK_SIMD PROPERTY STRINGS AUTO ON OFF) option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" OFF) @@ -35,6 +36,59 @@ if (NOT PERNIX_ENABLE_FALLBACK_STDPAR MATCHES "^(AUTO|ON|OFF)$") message(FATAL_ERROR "PERNIX_ENABLE_FALLBACK_STDPAR must be AUTO, ON, or OFF") endif () +string(TOUPPER "${PERNIX_ENABLE_FALLBACK_SIMD}" PERNIX_ENABLE_FALLBACK_SIMD) +set(PERNIX_ENABLE_FALLBACK_SIMD "${PERNIX_ENABLE_FALLBACK_SIMD}" CACHE STRING "Enable the C++26 std::simd fallback backend: OFF, AUTO, or ON." FORCE) +if (NOT PERNIX_ENABLE_FALLBACK_SIMD MATCHES "^(AUTO|ON|OFF)$") + message(FATAL_ERROR "PERNIX_ENABLE_FALLBACK_SIMD must be AUTO, ON, or OFF") +endif () + +set(PERNIX_BUILD_FALLBACK_SIMD OFF) +if (NOT PERNIX_ENABLE_FALLBACK_SIMD STREQUAL "OFF") + include(CheckCXXSourceCompiles) + if (DEFINED CMAKE_CXX_STANDARD) + set(PERNIX_SAVED_CMAKE_CXX_STANDARD "${CMAKE_CXX_STANDARD}") + set(PERNIX_CXX_STANDARD_WAS_DEFINED ON) + endif () + if (DEFINED CMAKE_CXX_STANDARD_REQUIRED) + set(PERNIX_SAVED_CMAKE_CXX_STANDARD_REQUIRED "${CMAKE_CXX_STANDARD_REQUIRED}") + set(PERNIX_CXX_STANDARD_REQUIRED_WAS_DEFINED ON) + endif () + set(CMAKE_CXX_STANDARD 26) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + check_cxx_source_compiles([=[ + #include + #include + + #if !defined(__cpp_lib_simd) + #error "The standard library does not provide C++26 std::simd" + #endif + + int main() { + std::simd values{}; + return static_cast(values.size() == 0); + } + ]=] PERNIX_HAS_CXX26_STDSIMD) + if (PERNIX_CXX_STANDARD_WAS_DEFINED) + set(CMAKE_CXX_STANDARD "${PERNIX_SAVED_CMAKE_CXX_STANDARD}") + else () + unset(CMAKE_CXX_STANDARD) + endif () + if (PERNIX_CXX_STANDARD_REQUIRED_WAS_DEFINED) + set(CMAKE_CXX_STANDARD_REQUIRED "${PERNIX_SAVED_CMAKE_CXX_STANDARD_REQUIRED}") + else () + unset(CMAKE_CXX_STANDARD_REQUIRED) + endif () + + if (PERNIX_HAS_CXX26_STDSIMD) + set(PERNIX_BUILD_FALLBACK_SIMD ON) + message(STATUS "PERNIX fallback SIMD backend enabled (C++26 std::simd available)") + elseif (PERNIX_ENABLE_FALLBACK_SIMD STREQUAL "ON") + message(FATAL_ERROR "PERNIX_ENABLE_FALLBACK_SIMD=ON requires compiler and standard-library support for C++26 std::simd") + else () + message(STATUS "PERNIX fallback SIMD backend disabled (C++26 std::simd unavailable)") + endif () +endif () + if (PERNIX_INSTALL_DOCS AND NOT PERNIX_ENABLE_DOXYGEN) message(FATAL_ERROR "PERNIX_INSTALL_DOCS requires PERNIX_ENABLE_DOXYGEN=ON") endif () diff --git a/README.md b/README.md index 27e78e2..85fdb82 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,9 @@ Common options: * `-DPERNIX_ENABLE_X86_AVX2=OFF`, `-DPERNIX_ENABLE_X86_BMI2=OFF`, `-DPERNIX_ENABLE_X86_AVX512VBMI=OFF`: disable specific x86 backends * `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF|AUTO|ON`: control the stdpar fallback backend. TBB is optional and enables parallel standard execution policies when found +* `-DPERNIX_ENABLE_FALLBACK_SIMD=OFF|AUTO|ON`: control the experimental C++26 + `std::simd` fallback backend. `AUTO` enables it when both the compiler and standard + library support `std::simd`; `ON` makes missing support a configuration error Install: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dc18d89..6567f8b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -93,7 +93,13 @@ if (NOT PERNIX_ENABLE_FALLBACK_STDPAR STREQUAL "OFF") endif () endif () -if (PERNIX_ENABLE_FALLBACK_SIMD) +if (PERNIX_BUILD_FALLBACK_SIMD) + target_sources(pernix PRIVATE + fallback/simd_compression.cpp + fallback/simd_decompression.cpp + ) + target_compile_features(pernix PUBLIC cxx_std_26) + target_compile_features(pernix_header_only INTERFACE cxx_std_26) target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_SIMD=1) target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_SIMD=1) endif () From 359df15986638811c0e872ebb643a859a9cd4431 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 16 Jul 2026 15:36:42 +0200 Subject: [PATCH 36/43] feat: add unpacking tables and extend functionality for AVX512 and AVX2 decompression --- include/pernix/arm64/neon/tables.h | 3 +- include/pernix/detail/api.hpp | 16 +- include/pernix/detail/pack_tables.h | 4 + include/pernix/detail/unpack_tables.h | 162 +++++++++ include/pernix/pernix.hpp | 96 +++--- include/pernix/x86/avx2/avx2_compression.h | 65 ++-- include/pernix/x86/avx2/avx2_decompression.h | 23 +- .../x86/avx512vbmi/avx512vbmi_decompression.h | 119 +++++-- include/pernix/x86/avx512vbmi/tables.h | 186 ---------- include/pernix/x86/avx512vbmi/unpacking.h | 322 ++++++------------ include/pernix/x86/bmi2/bmi2_compression.h | 18 +- include/pernix/x86/bmi2/bmi2_decompression.h | 14 +- tests/deterministic_correctness_tests.cpp | 45 ++- 13 files changed, 507 insertions(+), 566 deletions(-) create mode 100644 include/pernix/detail/pack_tables.h create mode 100644 include/pernix/detail/unpack_tables.h diff --git a/include/pernix/arm64/neon/tables.h b/include/pernix/arm64/neon/tables.h index dad2453..86dfc6a 100644 --- a/include/pernix/arm64/neon/tables.h +++ b/include/pernix/arm64/neon/tables.h @@ -1,10 +1,11 @@ #ifndef PERNIX_ARM64_NEON_TABLES_H #define PERNIX_ARM64_NEON_TABLES_H +#include + #include #include #include -#include namespace pernix::arm64::neon::internal { namespace detail { diff --git a/include/pernix/detail/api.hpp b/include/pernix/detail/api.hpp index 53aef32..04a11a4 100644 --- a/include/pernix/detail/api.hpp +++ b/include/pernix/detail/api.hpp @@ -114,9 +114,9 @@ inline pernix_status compress_block(const Backend backend, const u8 bit_width, c const auto kernel = [&] { if constexpr (std::is_same_v) { - return internal::select_compress_block_f32(backend, bit_width, block_size); + return pernix::internal::select_compress_block_f32(backend, bit_width, block_size); } else { - return internal::select_compress_block_f64(backend, bit_width, block_size); + return pernix::internal::select_compress_block_f64(backend, bit_width, block_size); } }(); if (!kernel) { @@ -144,9 +144,9 @@ inline pernix_status compress_blocks(const Backend backend, const u8 bit_width, const auto kernel = [&] { if constexpr (std::is_same_v) { - return internal::select_compress_blocks_f32(backend, bit_width, block_size); + return pernix::internal::select_compress_blocks_f32(backend, bit_width, block_size); } else { - return internal::select_compress_blocks_f64(backend, bit_width, block_size); + return pernix::internal::select_compress_blocks_f64(backend, bit_width, block_size); } }(); if (!kernel) { @@ -171,9 +171,9 @@ inline pernix_status decompress_block(const Backend backend, const u8 bit_width, const auto kernel = [&] { if constexpr (std::is_same_v) { - return internal::select_decompress_block_f32(backend, bit_width, block_size, sign_values); + return pernix::internal::select_decompress_block_f32(backend, bit_width, block_size, sign_values); } else { - return internal::select_decompress_block_f64(backend, bit_width, block_size, sign_values); + return pernix::internal::select_decompress_block_f64(backend, bit_width, block_size, sign_values); } }(); if (!kernel) { @@ -201,9 +201,9 @@ inline pernix_status decompress_blocks(const Backend backend, const u8 bit_width const auto kernel = [&] { if constexpr (std::is_same_v) { - return internal::select_decompress_blocks_f32(backend, bit_width, block_size, sign_values); + return pernix::internal::select_decompress_blocks_f32(backend, bit_width, block_size, sign_values); } else { - return internal::select_decompress_blocks_f64(backend, bit_width, block_size, sign_values); + return pernix::internal::select_decompress_blocks_f64(backend, bit_width, block_size, sign_values); } }(); if (!kernel) { diff --git a/include/pernix/detail/pack_tables.h b/include/pernix/detail/pack_tables.h new file mode 100644 index 0000000..23135ee --- /dev/null +++ b/include/pernix/detail/pack_tables.h @@ -0,0 +1,4 @@ +#ifndef PERNIX_PACK_TABLES_H +#define PERNIX_PACK_TABLES_H + +#endif // PERNIX_PACK_TABLES_H diff --git a/include/pernix/detail/unpack_tables.h b/include/pernix/detail/unpack_tables.h new file mode 100644 index 0000000..eb66927 --- /dev/null +++ b/include/pernix/detail/unpack_tables.h @@ -0,0 +1,162 @@ +#ifndef PERNIX_UNPACK_TABLES_H +#define PERNIX_UNPACK_TABLES_H + +#include + +#include +#include +#include +#include +#include + +namespace pernix::detail { +inline constexpr u8 inactive_lane = 0xff; + +namespace internal { +template +constexpr bool table_indices_are_valid(const std::array& table) { + return std::ranges::all_of(table, [](const u8 index) { return index == inactive_lane || index < Elements; }); +} + +template +constexpr std::array make_primary_permute() { + static_assert(Elements % IntWidth == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (usize entry = 0; entry < Elements / IntWidth; ++entry) { + const usize bit_start = entry * BitWidth; + const usize first_byte = bit_start / 8; + const usize base = entry * IntWidth; + + for (usize lane_byte = 0; lane_byte < IntWidth; ++lane_byte) { + table[base + lane_byte] = static_cast(first_byte + lane_byte); + } + } + + return table; +} + +template +constexpr std::array make_spill_permute() { + static_assert(Elements % IntWidth == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (usize entry = 0; entry < Elements / IntWidth; ++entry) { + const usize bit_start = entry * BitWidth; + const usize first_byte = bit_start / 8; + const usize bit_offset = bit_start % 8; + const usize base = entry * IntWidth; + + if (bit_offset + BitWidth > IntWidth * 8) { + table[base] = static_cast(first_byte + IntWidth); + } + } + + return table; +} + +template +constexpr std::array make_shift_right_magnitude() { + std::array table{}; + + for (usize entry = 0; entry < Elements; ++entry) { + const usize bit_offset = (entry * BitWidth) % 8; + table[entry] = static_cast(bit_offset); + } + + return table; +} + +template +constexpr std::array make_shift_right() { + auto table = make_shift_right_magnitude(); + for (ShiftType& shift : table) { + shift = -shift; + } + return table; +} + +template +constexpr std::array make_shift_left_for_spill() { + std::array table{}; + + for (usize entry = 0; entry < Elements; ++entry) { + const usize bit_offset = (entry * BitWidth) % 8; + const bool spills = bit_offset + BitWidth > IntWidth * 8; + table[entry] = spills ? static_cast(IntWidth * 8 - bit_offset) : 0; + } + + return table; +} +} // namespace internal + +template +struct unpack_table; + +template + requires std::signed_integral && (sizeof(ShiftType) == sizeof(u8)) && (BitWidth >= 1 && BitWidth <= 8) && (VectorBytes > 0) +struct unpack_table { + static constexpr usize shift_elements = VectorBytes / sizeof(ShiftType); + static constexpr u8 bit_width = BitWidth; + + alignas(64) static constexpr std::array primary_permute = + internal::make_primary_permute(); + alignas(64) static constexpr std::array spill_permute = + internal::make_spill_permute(); + alignas(64) static constexpr std::array right_shift = + internal::make_shift_right(); + alignas(64) static constexpr std::array right_shift_magnitude = + internal::make_shift_right_magnitude(); + alignas(64) static constexpr std::array left_shift_for_spill = + internal::make_shift_left_for_spill(); + static constexpr bool has_spill = std::ranges::any_of(left_shift_for_spill, [](const ShiftType shift) { return shift != 0; }); + + static_assert(internal::table_indices_are_valid(primary_permute)); + static_assert(internal::table_indices_are_valid(spill_permute)); +}; + +template + requires std::signed_integral && (sizeof(ShiftType) == sizeof(u16)) && (BitWidth >= 9 && BitWidth <= 16) && (VectorBytes > 0) +struct unpack_table { + static constexpr usize shift_elements = VectorBytes / sizeof(ShiftType); + static constexpr u8 bit_width = BitWidth; + + alignas(64) static constexpr std::array primary_permute = + internal::make_primary_permute(); + alignas(64) static constexpr std::array spill_permute = + internal::make_spill_permute(); + alignas(64) static constexpr std::array right_shift = + internal::make_shift_right(); + alignas(64) static constexpr std::array right_shift_magnitude = + internal::make_shift_right_magnitude(); + alignas(64) static constexpr std::array left_shift_for_spill = + internal::make_shift_left_for_spill(); + static constexpr bool has_spill = std::ranges::any_of(left_shift_for_spill, [](const ShiftType shift) { return shift != 0; }); + + static_assert(internal::table_indices_are_valid(primary_permute)); + static_assert(internal::table_indices_are_valid(spill_permute)); +}; + +template + requires std::signed_integral && (sizeof(ShiftType) == sizeof(u32)) && (BitWidth >= 17 && BitWidth <= 24) && + (VectorBytes > 0) +struct unpack_table { + static constexpr usize shift_elements = VectorBytes / sizeof(ShiftType); + static constexpr u8 bit_width = BitWidth; + + alignas(64) static constexpr std::array primary_permute = + internal::make_primary_permute(); + alignas(64) static constexpr std::array right_shift = + internal::make_shift_right(); + alignas(64) static constexpr std::array right_shift_magnitude = + internal::make_shift_right_magnitude(); + + static_assert(internal::table_indices_are_valid(primary_permute)); +}; +} // namespace pernix::detail + +#endif // PERNIX_UNPACK_TABLES_H diff --git a/include/pernix/pernix.hpp b/include/pernix/pernix.hpp index 78fc1de..602e13a 100644 --- a/include/pernix/pernix.hpp +++ b/include/pernix/pernix.hpp @@ -95,64 +95,64 @@ __always_inline Status validate_decompress_spans(const u8 bit_width, const u32 b } } // namespace detail -__always_inline Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, - const float inverse_scale, std::span output) { +PERNIX_API static Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, std::span output) { if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::compress_block(backend, bit_width, block_size, input.data(), inverse_scale, output.data()); } -__always_inline Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, - const double inverse_scale, std::span output) { +PERNIX_API static Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, std::span output) { if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::compress_block(backend, bit_width, block_size, input.data(), inverse_scale, output.data()); } -__always_inline Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, - const float scale, std::span output, const bool sign_values = true) { +PERNIX_API static Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float scale, std::span output, const bool sign_values = true) { if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } -__always_inline Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, - const double scale, std::span output, const bool sign_values = true) { +PERNIX_API static Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double scale, std::span output, const bool sign_values = true) { if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { return status; } return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); } -__always_inline Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, - const float inverse_scale, std::span output, const u32 blocks) { +PERNIX_API static Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, std::span output, const u32 blocks) { if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } return detail::compress_blocks(backend, bit_width, block_size, input.data(), inverse_scale, output.data(), blocks); } -__always_inline Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, - const double inverse_scale, std::span output, const u32 blocks) { +PERNIX_API static Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, std::span output, const u32 blocks) { if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } return detail::compress_blocks(backend, bit_width, block_size, input.data(), inverse_scale, output.data(), blocks); } -__always_inline Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, - const float scale, std::span output, const u32 blocks, const bool sign_values = true) { +PERNIX_API static Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float scale, std::span output, const u32 blocks, const bool sign_values = true) { if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, sign_values); } -__always_inline Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, - const double scale, std::span output, const u32 blocks, const bool sign_values = true) { +PERNIX_API static Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double scale, std::span output, const u32 blocks, const bool sign_values = true) { if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { return status; } @@ -160,84 +160,84 @@ __always_inline Status decompress_blocks(Backend backend, const u8 bit_width, co } // convenience overloads without backend (defaults to Auto) -__always_inline Status compress_block(const u8 bit_width, const u32 block_size, const std::span input, - const float inverse_scale, const std::span output) { +PERNIX_API static Status compress_block(const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, const std::span output) { return compress_block(Backend::Auto, bit_width, block_size, input, inverse_scale, output); } -__always_inline Status compress_block(const u8 bit_width, const u32 block_size, const std::span input, - const double inverse_scale, const std::span output) { +PERNIX_API static Status compress_block(const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, const std::span output) { return compress_block(Backend::Auto, bit_width, block_size, input, inverse_scale, output); } -__always_inline Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, const float scale, - const std::span output, const bool sign_values = true) { +PERNIX_API static Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, const float scale, + const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); } -__always_inline Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, const double scale, - const std::span output, const bool sign_values = true) { +PERNIX_API static Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, const double scale, + const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); } -__always_inline Status compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, - const float inverse_scale, const std::span output, const u32 blocks) { +PERNIX_API static Status compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, block_size, input, inverse_scale, output, blocks); } -__always_inline Status compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, - const double inverse_scale, const std::span output, const u32 blocks) { +PERNIX_API static Status compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, block_size, input, inverse_scale, output, blocks); } -__always_inline Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, const float scale, - const std::span output, const u32 blocks, const bool sign_values = true) { +PERNIX_API static Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, const float scale, + const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); } -__always_inline Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, const double scale, - const std::span output, const u32 blocks, const bool sign_values = true) { +PERNIX_API static Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, const double scale, + const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); } // convenience overloads without backend and without block_size (defaults to 64) -__always_inline Status compress_block(const u8 bit_width, const std::span input, const float inverse_scale, - const std::span output) { +PERNIX_API static Status compress_block(const u8 bit_width, const std::span input, const float inverse_scale, + const std::span output) { return compress_block(Backend::Auto, bit_width, 64, input, inverse_scale, output); } -__always_inline Status compress_block(const u8 bit_width, const std::span input, const double inverse_scale, - const std::span output) { +PERNIX_API static Status compress_block(const u8 bit_width, const std::span input, const double inverse_scale, + const std::span output) { return compress_block(Backend::Auto, bit_width, 64, input, inverse_scale, output); } -__always_inline Status decompress_block(const u8 bit_width, const std::span input, const float scale, - const std::span output, const bool sign_values = true) { +PERNIX_API static Status decompress_block(const u8 bit_width, const std::span input, const float scale, + const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); } -__always_inline Status decompress_block(const u8 bit_width, const std::span input, const double scale, - const std::span output, const bool sign_values = true) { +PERNIX_API static Status decompress_block(const u8 bit_width, const std::span input, const double scale, + const std::span output, const bool sign_values = true) { return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); } -__always_inline Status compress_blocks(const u8 bit_width, const std::span input, const float inverse_scale, - const std::span output, const u32 blocks) { +PERNIX_API static Status compress_blocks(const u8 bit_width, const std::span input, const float inverse_scale, + const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, 64, input, inverse_scale, output, blocks); } -__always_inline Status compress_blocks(const u8 bit_width, const std::span input, const double inverse_scale, - const std::span output, const u32 blocks) { +PERNIX_API static Status compress_blocks(const u8 bit_width, const std::span input, const double inverse_scale, + const std::span output, const u32 blocks) { return compress_blocks(Backend::Auto, bit_width, 64, input, inverse_scale, output, blocks); } -__always_inline Status decompress_blocks(const u8 bit_width, const std::span input, const float scale, - const std::span output, const u32 blocks, const bool sign_values = true) { +PERNIX_API static Status decompress_blocks(const u8 bit_width, const std::span input, const float scale, + const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); } -__always_inline Status decompress_blocks(const u8 bit_width, const std::span input, const double scale, - const std::span output, const u32 blocks, const bool sign_values = true) { +PERNIX_API static Status decompress_blocks(const u8 bit_width, const std::span input, const double scale, + const std::span output, const u32 blocks, const bool sign_values = true) { return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); } } // namespace pernix diff --git a/include/pernix/x86/avx2/avx2_compression.h b/include/pernix/x86/avx2/avx2_compression.h index ce8fd98..cb07a0b 100644 --- a/include/pernix/x86/avx2/avx2_compression.h +++ b/include/pernix/x86/avx2/avx2_compression.h @@ -213,9 +213,10 @@ __always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { } __always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i& input) { - const __m256i zero = _mm256_setzero_si256(); + const __m256i zero = _mm256_setzero_si256(); + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(0x0f)); - const __m256i packed16 = _mm256_packus_epi32(input, zero); + const __m256i packed16 = _mm256_packus_epi32(masked, zero); const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); const __m256i packed8 = _mm256_packus_epi16(permuted, zero); @@ -297,9 +298,10 @@ __always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i& input) -> __m256 template requires(BIT_WIDTH >= 5 && BIT_WIDTH <= 7) __always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i { - const __m256i zero = _mm256_setzero_si256(); + const __m256i zero = _mm256_setzero_si256(); + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32((1 << BIT_WIDTH) - 1)); - const __m256i packed16 = _mm256_packus_epi32(input, zero); + const __m256i packed16 = _mm256_packus_epi32(masked, zero); const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); const __m256i packed8 = _mm256_packus_epi16(permuted, zero); @@ -415,19 +417,6 @@ __m256i mm256_pack_epi32_avx2(const __m256i& input) { } return _mm256_setzero_si256(); } - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -void store_packed_epi32_fallback_8(const __m256i& input, u8* __restrict__ output) { - alignas(32) std::array lanes{}; - _mm256_storeu_si256(reinterpret_cast<__m256i*>(lanes.data()), input); - - std::vector block_values(lanes.size()); - for (usize i = 0; i < lanes.size(); ++i) { - block_values[i] = static_cast(lanes[i]); - } - pack_epi32_fallback(block_values, output); -} } // namespace internal /** @@ -461,20 +450,25 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scal const __m256 source = _mm256_loadu_ps(input); const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); - internal::store_packed_epi32_fallback_8(packed_input, output); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + std::memcpy(output, &packed, BIT_WIDTH); input += 8; output += BIT_WIDTH; } if constexpr (remaining) { - std::vector block_values(remaining); -#pragma GCC unroll 8 - for (u32 i = 0; i < remaining; i++) { - block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], scale))); - } - - internal::pack_epi32_fallback(block_values, output); + // std::vector block_values(remaining); + // #pragma GCC unroll 8 + // for (u32 i = 0; i < remaining; i++) { + // block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], + // scale))); + // } + // + // internal::pack_epi32_fallback(block_values, output); + + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + internal::quantize_and_pack_fallback(std::span(input, remaining), scale, remaining, std::span(output, tail_bytes)); } return 0; @@ -515,19 +509,24 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scal __m256i combined = _mm256_castsi128_si256(quantized1); combined = _mm256_inserti128_si256(combined, quantized2, 1); const __m256i packed_input = internal::mm256_clamp_signed_epi32(combined); - internal::store_packed_epi32_fallback_8(packed_input, output); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + std::memcpy(output, &packed, BIT_WIDTH); input += 8; output += BIT_WIDTH; } if constexpr (remaining) { - std::vector block_values(remaining); -#pragma GCC unroll 8 - for (u32 i = 0; i < remaining; i++) { - block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], scale))); - } - - internal::pack_epi32_fallback(block_values, output); + // std::vector block_values(remaining); + // #pragma GCC unroll 8 + // for (u32 i = 0; i < remaining; i++) { + // block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], + // scale))); + // } + // + // internal::pack_epi32_fallback(block_values, output); + + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + internal::quantize_and_pack_fallback(std::span(input, remaining), scale, remaining, std::span(output, tail_bytes)); } return 0; diff --git a/include/pernix/x86/avx2/avx2_decompression.h b/include/pernix/x86/avx2/avx2_decompression.h index 11e8978..94d1fc4 100644 --- a/include/pernix/x86/avx2/avx2_decompression.h +++ b/include/pernix/x86/avx2/avx2_decompression.h @@ -211,10 +211,13 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f32 sc } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (u32 i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi32(tail_values[i], scale); - } + // const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); + // for (u32 i = 0; i < remaining; i++) { + // output[i] = internal::dequantize_epi32(tail_values[i], scale); + // } + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + internal::unpack_and_dequantize_fallback(std::span(input, tail_bytes), remaining, scale, + std::span(output, remaining)); } return 0; @@ -260,10 +263,14 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f64 sc } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (u32 i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi64(tail_values[i], scale); - } + // const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); + // for (u32 i = 0; i < remaining; i++) { + // output[i] = internal::dequantize_epi64(tail_values[i], scale); + // } + + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + internal::unpack_and_dequantize_fallback(std::span(input, tail_bytes), remaining, scale, + std::span(output, remaining)); } return 0; } diff --git a/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h index ba1c6c9..22a1fb7 100644 --- a/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h +++ b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h @@ -26,6 +26,51 @@ __always_inline __m512d mm512_dequantize_epi64(const __m512i& input, const __m51 return _mm512_mul_pd(converted, scale); } +template +__always_inline __m512i mm512_extend_epi8_epi32(const __m128i input) { + if constexpr (SIGN_VALUES) { + return _mm512_cvtepi8_epi32(input); + } else { + return _mm512_cvtepu8_epi32(input); + } +} + +template +__always_inline __m512i mm512_extend_epi8_epi64(const __m128i input) { + if constexpr (SIGN_VALUES) { + return _mm512_cvtepi8_epi64(input); + } else { + return _mm512_cvtepu8_epi64(input); + } +} + +template +__always_inline __m512i mm512_extend_epi16_epi32(const __m256i input) { + if constexpr (SIGN_VALUES) { + return _mm512_cvtepi16_epi32(input); + } else { + return _mm512_cvtepu16_epi32(input); + } +} + +template +__always_inline __m512i mm512_extend_epi16_epi64(const __m128i input) { + if constexpr (SIGN_VALUES) { + return _mm512_cvtepi16_epi64(input); + } else { + return _mm512_cvtepu16_epi64(input); + } +} + +template +__always_inline __m256i mm256_extend_epi16_epi32(const __m128i input) { + if constexpr (SIGN_VALUES) { + return _mm256_cvtepi16_epi32(input); + } else { + return _mm256_cvtepu16_epi32(input); + } +} + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { @@ -44,10 +89,10 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted1 = _mm512_cvtepi8_epi32(_mm512_castsi512_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 1)); - const __m512i converted3 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 2)); - const __m512i converted4 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 3)); + const __m512i converted1 = mm512_extend_epi8_epi32(_mm512_castsi512_si128(unpacked)); + const __m512i converted2 = mm512_extend_epi8_epi32(_mm512_extracti64x2_epi64(unpacked, 1)); + const __m512i converted3 = mm512_extend_epi8_epi32(_mm512_extracti64x2_epi64(unpacked, 2)); + const __m512i converted4 = mm512_extend_epi8_epi32(_mm512_extracti64x2_epi64(unpacked, 3)); const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); @@ -68,8 +113,8 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted1 = _mm512_cvtepi8_epi32(_mm256_castsi256_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi8_epi32(_mm256_extracti128_si256(unpacked, 1)); + const __m512i converted1 = mm512_extend_epi8_epi32(_mm256_castsi256_si128(unpacked)); + const __m512i converted2 = mm512_extend_epi8_epi32(_mm256_extracti128_si256(unpacked, 1)); const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); @@ -85,7 +130,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted = _mm512_cvtepi8_epi32(unpacked); + const __m512i converted = mm512_extend_epi8_epi32(unpacked); const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); @@ -99,7 +144,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted = _mm512_cvtepi8_epi32(unpacked); + const __m512i converted = mm512_extend_epi8_epi32(unpacked); const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); @@ -132,14 +177,14 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ const __m128i extracted3 = _mm512_extracti64x2_epi64(unpacked, 2); const __m128i extracted4 = _mm512_extracti64x2_epi64(unpacked, 3); - const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); - const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); - const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); - const __m512i converted5 = _mm512_cvtepi8_epi64(extracted3); - const __m512i converted6 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted3, 8)); - const __m512i converted7 = _mm512_cvtepi8_epi64(extracted4); - const __m512i converted8 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted4, 8)); + const __m512i converted1 = mm512_extend_epi8_epi64(extracted1); + const __m512i converted2 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted1, 8)); + const __m512i converted3 = mm512_extend_epi8_epi64(extracted2); + const __m512i converted4 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted2, 8)); + const __m512i converted5 = mm512_extend_epi8_epi64(extracted3); + const __m512i converted6 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted3, 8)); + const __m512i converted7 = mm512_extend_epi8_epi64(extracted4); + const __m512i converted8 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted4, 8)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -170,10 +215,10 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ const __m128i extracted1 = _mm256_castsi256_si128(unpacked); const __m128i extracted2 = _mm256_extracti64x2_epi64(unpacked, 1); - const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); - const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); - const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); + const __m512i converted1 = mm512_extend_epi8_epi64(extracted1); + const __m512i converted2 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted1, 8)); + const __m512i converted3 = mm512_extend_epi8_epi64(extracted2); + const __m512i converted4 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted2, 8)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -193,8 +238,8 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); + const __m512i converted1 = mm512_extend_epi8_epi64(unpacked); + const __m512i converted2 = mm512_extend_epi8_epi64(_mm_srli_si128(unpacked, 8)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -210,8 +255,8 @@ __always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict_ const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); + const __m512i converted1 = mm512_extend_epi8_epi64(unpacked); + const __m512i converted2 = mm512_extend_epi8_epi64(_mm_srli_si128(unpacked, 8)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -245,8 +290,8 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted1 = _mm512_cvtepi16_epi32(_mm512_castsi512_si256(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi32(_mm512_extracti32x8_epi32(unpacked, 1)); + const __m512i converted1 = mm512_extend_epi16_epi32(_mm512_castsi512_si256(unpacked)); + const __m512i converted2 = mm512_extend_epi16_epi32(_mm512_extracti32x8_epi32(unpacked, 1)); const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); @@ -263,7 +308,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted = _mm512_cvtepi16_epi32(unpacked); + const __m512i converted = mm512_extend_epi16_epi32(unpacked); const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); _mm512_storeu_ps(output, dequantized); @@ -276,7 +321,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); - const __m256i converted = _mm256_cvtepi16_epi32(unpacked); + const __m256i converted = mm256_extend_epi16_epi32(unpacked); const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); _mm256_storeu_ps(output, dequantized); @@ -289,7 +334,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); - const __m256i converted = _mm256_cvtepi16_epi32(unpacked); + const __m256i converted = mm256_extend_epi16_epi32(unpacked); const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); mm256_storeu_elements_ps(output, remaining_elements, dequantized); @@ -316,10 +361,10 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted1 = _mm512_cvtepi16_epi64(_mm512_castsi512_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 1)); - const __m512i converted3 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 2)); - const __m512i converted4 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 3)); + const __m512i converted1 = mm512_extend_epi16_epi64(_mm512_castsi512_si128(unpacked)); + const __m512i converted2 = mm512_extend_epi16_epi64(_mm512_extracti64x2_epi64(unpacked, 1)); + const __m512i converted3 = mm512_extend_epi16_epi64(_mm512_extracti64x2_epi64(unpacked, 2)); + const __m512i converted4 = mm512_extend_epi16_epi64(_mm512_extracti64x2_epi64(unpacked, 3)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -340,8 +385,8 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted1 = _mm512_cvtepi16_epi64(_mm256_castsi256_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi64(_mm256_extracti64x2_epi64(unpacked, 1)); + const __m512i converted1 = mm512_extend_epi16_epi64(_mm256_castsi256_si128(unpacked)); + const __m512i converted2 = mm512_extend_epi16_epi64(_mm256_extracti64x2_epi64(unpacked, 1)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -357,7 +402,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted = _mm512_cvtepi16_epi64(unpacked); + const __m512i converted = mm512_extend_epi16_epi64(unpacked); const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); @@ -371,7 +416,7 @@ __always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted = _mm512_cvtepi16_epi64(unpacked); + const __m512i converted = mm512_extend_epi16_epi64(unpacked); const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); diff --git a/include/pernix/x86/avx512vbmi/tables.h b/include/pernix/x86/avx512vbmi/tables.h index d708e10..976c873 100644 --- a/include/pernix/x86/avx512vbmi/tables.h +++ b/include/pernix/x86/avx512vbmi/tables.h @@ -629,192 +629,6 @@ struct pack_tables_avx512_24 { static __always_inline Vec get_shift3() { return load_table(shift3); } }; -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct unpack_tables_avx512_8 { -private: - alignas(64) inline static constexpr std::array permute1 = [] { - std::array table{}; - std::ranges::fill(table, -1); - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - - table[entry] = static_cast(first_byte); - } - - return table; - }(); - - alignas(64) inline static constexpr std::array permute2 = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t bit_offset = bit_start % 8; - - if (bit_offset + BIT_WIDTH > 8) { - table[entry] = static_cast(first_byte + 1); - } - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift1 = [] { - std::array table{}; - - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8; - - table[entry] = static_cast(bit_offset); - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift2 = [] { - std::array table{}; - - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - const size_t spill_bits = (bit_offset + BIT_WIDTH > 8u) ? (bit_offset + BIT_WIDTH - 8u) : 0u; - - table[entry] = spill_bits ? static_cast(8 - bit_offset) : 0; - } - - return table; - }(); - -public: - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } - - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } -}; - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct unpack_tables_avx512_16 { -private: - alignas(64) inline static constexpr std::array permute1 = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t base = entry * 2; - - table[base] = static_cast(first_byte); - table[base + 1] = static_cast(first_byte + 1); - } - - return table; - }(); - - alignas(64) inline static constexpr std::array permute2 = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t bit_offset = bit_start % 8; - const size_t base = entry * 2; - - if (bit_offset + BIT_WIDTH > 16) { - table[base] = static_cast(first_byte + 2); - } - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift1 = [] { - std::array table{}; - - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - - // Right-shift the 16-bit chunk so the value starts at bit 0. - table[entry] = static_cast(bit_offset); - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift2 = [] { - std::array table{}; - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - const size_t spill_bits = (bit_offset + BIT_WIDTH > 16u) ? (bit_offset + BIT_WIDTH - 16u) : 0u; - - // Move spill bits from byte3 to their final bit positions before merge. - table[entry] = spill_bits ? static_cast(16u - bit_offset) : 0; - } - - return table; - }(); - -public: - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } - - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } -}; - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct unpack_tables_avx512_24 { -private: - alignas(64) inline static constexpr std::array permute = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 16; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_end = bit_start + BIT_WIDTH - 1; - - const size_t first_byte = bit_start / 8; - const size_t last_byte = bit_end / 8; - - const size_t base = entry * 4; - - for (size_t byte = first_byte; byte <= last_byte; ++byte) { - table[base + (byte - first_byte)] = static_cast(byte); - } - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift = [] { - std::array table{}; - - for (size_t entry = 0; entry < 16; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - table[entry] = static_cast(32u - BIT_WIDTH - (bit_start % 8u)); - } - - return table; - }(); - -public: - static __always_inline Vec get_permute() { return load_table(permute); } - static __always_inline Vec get_shift() { return load_table(shift); } -}; } // namespace pernix::internal #endif // PERNIX_AVX512VBMI_TABLES_H diff --git a/include/pernix/x86/avx512vbmi/unpacking.h b/include/pernix/x86/avx512vbmi/unpacking.h index 06c29b3..1426460 100644 --- a/include/pernix/x86/avx512vbmi/unpacking.h +++ b/include/pernix/x86/avx512vbmi/unpacking.h @@ -1,6 +1,7 @@ #ifndef PERNIX_AVX512VBMI_UNPACKING_H #define PERNIX_AVX512VBMI_UNPACKING_H +#include #include #include @@ -56,73 +57,29 @@ __always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i& input) { if constexpr (BIT_WIDTH == 8) { return input; } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask16>(_mm_cvtsi128_si64(input)); - const __m128i source = _mm_movm_epi8(value); - const __m128i unpacked = _mm_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m128i values_shift0 = input; - __m128i values_shift2 = _mm_srli_epi16(values_shift0, 2); - const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); - const __m128i values_shift6 = _mm_srli_epi16(values_shift0, 6); - - __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - - interleave_tmp = _mm_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm_unpacklo_epi64(interleave_tmp, values_shift2); - - interleave_tmp = _mm_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); - - values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0303)); - - if constexpr (SIGN_VALUES) { - values_shift0 = _mm_srai_epi8(_mm_slli_epi8(values_shift0, 6), 6); - } - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m128i values_shift0 = input; - const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); - - const __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); - - values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0F0F)); - - if constexpr (SIGN_VALUES) { - values_shift0 = _mm_srai_epi8(_mm_slli_epi8(values_shift0, 4), 4); - } - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m128i permuted1 = _mm_permutexvar_epi8(tables::get_permute1(), input); - const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); - - const __m128i shifted1 = _mm_srlv_epi8(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask16 spill_mask = _mm_cmpneq_epi8_mask(tables::get_shift2(), _mm_setzero_si128()); - __m128i combined = _mm_or_si128(shifted1, _mm_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr u32 shift = 8 - BIT_WIDTH; - combined = _mm_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm_srai_epi8(combined, shift); - } else { - combined = _mm_srli_epi8(combined, shift); - } + using tables = detail::unpack_table; + + const __m128i permuted = _mm_permutexvar_epi8(load_table<__m128i>(tables::primary_permute), input); + const __m128i right_shift = load_table<__m128i>(tables::right_shift_magnitude); + __m128i combined = _mm_srlv_epi8(permuted, right_shift); + + if constexpr (tables::has_spill) { + const __m128i spill_permuted = _mm_permutexvar_epi8(load_table<__m128i>(tables::spill_permute), input); + const __m128i spill_shift = load_table<__m128i>(tables::left_shift_for_spill); + const __m128i spill_values = _mm_sllv_epi8(spill_permuted, spill_shift); + const __mmask16 spill_mask = _mm_cmpneq_epi8_mask(spill_shift, _mm_setzero_si128()); + combined = _mm_or_si128(combined, _mm_maskz_mov_epi8(spill_mask, spill_values)); + } - return combined; + constexpr u32 shift = 8 - BIT_WIDTH; + combined = _mm_slli_epi8(combined, shift); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + combined = _mm_srai_epi8(combined, shift); + } else { + combined = _mm_srli_epi8(combined, shift); } + + return combined; } } @@ -132,15 +89,16 @@ __always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i& input) { if constexpr (BIT_WIDTH == 16) { return input; } else { - using tables = unpack_tables_avx512_16; + using tables = detail::unpack_table; - const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute1(), input); + const __m128i permuted = _mm_permutexvar_epi8(load_table<__m128i>(tables::primary_permute), input); - __m128i shifted = _mm_srlv_epi16(permuted, tables::get_shift1()); + const __m128i right_shift = load_table<__m128i>(tables::right_shift_magnitude); + __m128i shifted = _mm_srlv_epi16(permuted, right_shift); - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); - const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); + if constexpr (tables::has_spill) { + const __m128i permuted2 = _mm_permutexvar_epi8(load_table<__m128i>(tables::spill_permute), input); + const __m128i shifted2 = _mm_sllv_epi16(permuted2, load_table<__m128i>(tables::left_shift_for_spill)); shifted = _mm_or_si128(shifted, shifted2); } @@ -159,12 +117,14 @@ __always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i& input) { template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i& input) { - using tables = unpack_tables_avx512_24; + using tables = detail::unpack_table; - const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute(), input); + const __m128i permuted = _mm_permutexvar_epi8(load_table<__m128i>(tables::primary_permute), input); + const __m128i right_shift = load_table<__m128i>(tables::right_shift_magnitude); constexpr u32 shift = 32 - BIT_WIDTH; - __m128i shifted = _mm_sllv_epi32(permuted, tables::get_shift()); + __m128i shifted = _mm_srlv_epi32(permuted, right_shift); + shifted = _mm_slli_epi32(shifted, shift); if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { shifted = _mm_srai_epi32(shifted, shift); } else { @@ -226,73 +186,29 @@ __always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i& input) if constexpr (BIT_WIDTH == 8) { return input; } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask32>(_mm_cvtsi128_si64(_mm256_castsi256_si128(input))); - const __m256i source = _mm256_movm_epi8(value); - const __m256i unpacked = _mm256_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m256i values_shift0 = input; - __m256i values_shift2 = _mm256_srli_epi16(values_shift0, 2); - const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); - const __m256i values_shift6 = _mm256_srli_epi16(values_shift0, 6); - - __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); - - interleave_tmp = _mm256_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm256_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm256_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); - - interleave_tmp = _mm256_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm256_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); - values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); - - values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0303)); - - if constexpr (SIGN_VALUES) { - values_shift0 = _mm256_srai_epi8(_mm256_slli_epi8(values_shift0, 6), 6); - } - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m256i values_shift0 = input; - const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); - - __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); - values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); - - values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0F0F)); - - if constexpr (SIGN_VALUES) { - values_shift0 = _mm256_srai_epi8(_mm256_slli_epi8(values_shift0, 4), 4); - } - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m256i permuted1 = _mm256_permutexvar_epi8(tables::get_permute1(), input); - const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); - - const __m256i shifted1 = _mm256_srlv_epi8(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask32 spill_mask = _mm256_cmpneq_epi8_mask(tables::get_shift2(), _mm256_setzero_si256()); - __m256i combined = _mm256_or_si256(shifted1, _mm256_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr u32 shift = 8 - BIT_WIDTH; - combined = _mm256_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm256_srai_epi8(combined, shift); - } else { - combined = _mm256_srli_epi8(combined, shift); - } + using tables = detail::unpack_table; + + const __m256i permuted = _mm256_permutexvar_epi8(load_table<__m256i>(tables::primary_permute), input); + const __m256i right_shift = load_table<__m256i>(tables::right_shift_magnitude); + __m256i combined = _mm256_srlv_epi8(permuted, right_shift); + + if constexpr (tables::has_spill) { + const __m256i spill_permuted = _mm256_permutexvar_epi8(load_table<__m256i>(tables::spill_permute), input); + const __m256i spill_shift = load_table<__m256i>(tables::left_shift_for_spill); + const __m256i spill_values = _mm256_sllv_epi8(spill_permuted, spill_shift); + const __mmask32 spill_mask = _mm256_cmpneq_epi8_mask(spill_shift, _mm256_setzero_si256()); + combined = _mm256_or_si256(combined, _mm256_maskz_mov_epi8(spill_mask, spill_values)); + } - return combined; + constexpr u32 shift = 8 - BIT_WIDTH; + combined = _mm256_slli_epi8(combined, shift); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + combined = _mm256_srai_epi8(combined, shift); + } else { + combined = _mm256_srli_epi8(combined, shift); } + + return combined; } } @@ -302,15 +218,16 @@ __always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i& input if constexpr (BIT_WIDTH == 16) { return input; } else { - using tables = unpack_tables_avx512_16; + using tables = detail::unpack_table; - const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute1(), input); + const __m256i permuted = _mm256_permutexvar_epi8(load_table<__m256i>(tables::primary_permute), input); - __m256i shifted = _mm256_srlv_epi16(permuted, tables::get_shift1()); + const __m256i right_shift = load_table<__m256i>(tables::right_shift_magnitude); + __m256i shifted = _mm256_srlv_epi16(permuted, right_shift); - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); - const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); + if constexpr (tables::has_spill) { + const __m256i permuted2 = _mm256_permutexvar_epi8(load_table<__m256i>(tables::spill_permute), input); + const __m256i shifted2 = _mm256_sllv_epi16(permuted2, load_table<__m256i>(tables::left_shift_for_spill)); shifted = _mm256_or_si256(shifted, shifted2); } @@ -329,12 +246,14 @@ __always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i& input template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i& input) { - using tables = unpack_tables_avx512_24; + using tables = detail::unpack_table; - const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute(), input); + const __m256i permuted = _mm256_permutexvar_epi8(load_table<__m256i>(tables::primary_permute), input); + const __m256i right_shift = load_table<__m256i>(tables::right_shift_magnitude); constexpr u32 shift = 32 - BIT_WIDTH; - __m256i shifted = _mm256_sllv_epi32(permuted, tables::get_shift()); + __m256i shifted = _mm256_srlv_epi32(permuted, right_shift); + shifted = _mm256_slli_epi32(shifted, shift); if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { shifted = _mm256_srai_epi32(shifted, shift); } else { @@ -396,73 +315,29 @@ __always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i& input) if constexpr (BIT_WIDTH == 8) { return input; } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask64>(_mm_cvtsi128_si64(_mm512_castsi512_si128(input))); - const __m512i source = _mm512_movm_epi8(value); - const __m512i unpacked = _mm512_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m512i values_shift0 = input; - __m512i values_shift2 = _mm512_srli_epi16(values_shift0, 2); - const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); - const __m512i values_shift6 = _mm512_srli_epi16(values_shift0, 6); - - __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); - - interleave_tmp = _mm512_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm512_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm512_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); - - interleave_tmp = _mm512_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm512_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x88); - values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); - - values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0303)); - - if constexpr (SIGN_VALUES) { - values_shift0 = _mm512_srai_epi8(_mm512_slli_epi8(values_shift0, 6), 6); - } - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m512i values_shift0 = input; - const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); - - __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x44); - values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); - - values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0F0F)); - - if constexpr (SIGN_VALUES) { - values_shift0 = _mm512_srai_epi8(_mm512_slli_epi8(values_shift0, 4), 4); - } - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m512i permuted1 = _mm512_permutexvar_epi8(tables::get_permute1(), input); - const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); - - const __m512i shifted1 = _mm512_srlv_epi8(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask64 spill_mask = _mm512_cmpneq_epi8_mask(tables::get_shift2(), _mm512_setzero_si512()); - __m512i combined = _mm512_or_si512(shifted1, _mm512_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr u32 shift = 8 - BIT_WIDTH; - combined = _mm512_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm512_srai_epi8(combined, shift); - } else { - combined = _mm512_srli_epi8(combined, shift); - } + using tables = detail::unpack_table; + + const __m512i permuted = _mm512_permutexvar_epi8(load_table<__m512i>(tables::primary_permute), input); + const __m512i right_shift = load_table<__m512i>(tables::right_shift_magnitude); + __m512i combined = _mm512_srlv_epi8(permuted, right_shift); + + if constexpr (tables::has_spill) { + const __m512i spill_permuted = _mm512_permutexvar_epi8(load_table<__m512i>(tables::spill_permute), input); + const __m512i spill_shift = load_table<__m512i>(tables::left_shift_for_spill); + const __m512i spill_values = _mm512_sllv_epi8(spill_permuted, spill_shift); + const __mmask64 spill_mask = _mm512_cmpneq_epi8_mask(spill_shift, _mm512_setzero_si512()); + combined = _mm512_or_si512(combined, _mm512_maskz_mov_epi8(spill_mask, spill_values)); + } - return combined; + constexpr u32 shift = 8 - BIT_WIDTH; + combined = _mm512_slli_epi8(combined, shift); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + combined = _mm512_srai_epi8(combined, shift); + } else { + combined = _mm512_srli_epi8(combined, shift); } + + return combined; } } @@ -472,14 +347,15 @@ __always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i& input if constexpr (BIT_WIDTH == 16) { return input; } else { - using tables = unpack_tables_avx512_16; + using tables = detail::unpack_table; - const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute1(), input); - __m512i shifted = _mm512_srlv_epi16(permuted, tables::get_shift1()); + const __m512i permuted = _mm512_permutexvar_epi8(load_table<__m512i>(tables::primary_permute), input); + const __m512i right_shift = load_table<__m512i>(tables::right_shift_magnitude); + __m512i shifted = _mm512_srlv_epi16(permuted, right_shift); - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); - const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); + if constexpr (tables::has_spill) { + const __m512i permuted2 = _mm512_permutexvar_epi8(load_table<__m512i>(tables::spill_permute), input); + const __m512i shifted2 = _mm512_sllv_epi16(permuted2, load_table<__m512i>(tables::left_shift_for_spill)); shifted = _mm512_or_si512(shifted, shifted2); } @@ -498,12 +374,14 @@ __always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i& input template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) __always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i& input) { - using tables = unpack_tables_avx512_24; + using tables = detail::unpack_table; - const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute(), input); - __m512i shifted = _mm512_sllv_epi32(permuted, tables::get_shift()); + const __m512i permuted = _mm512_permutexvar_epi8(load_table<__m512i>(tables::primary_permute), input); + const __m512i right_shift = load_table<__m512i>(tables::right_shift_magnitude); + __m512i shifted = _mm512_srlv_epi32(permuted, right_shift); constexpr u32 shift = 32 - BIT_WIDTH; + shifted = _mm512_slli_epi32(shifted, shift); if constexpr (SIGN_VALUES) { shifted = _mm512_srai_epi32(shifted, shift); } else { diff --git a/include/pernix/x86/bmi2/bmi2_compression.h b/include/pernix/x86/bmi2/bmi2_compression.h index 5343f28..645b273 100644 --- a/include/pernix/x86/bmi2/bmi2_compression.h +++ b/include/pernix/x86/bmi2/bmi2_compression.h @@ -186,13 +186,8 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scal } if constexpr (remaining) { - std::vector block_values(remaining); -#pragma GCC unroll 8 - for (u32 i = 0; i < remaining; i++) { - block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], scale))); - } - - internal::pack_epi32_fallback(block_values, output); + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + internal::quantize_and_pack_fallback(std::span(input, remaining), scale, remaining, std::span(output, tail_bytes)); } return 0; @@ -237,13 +232,8 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scal } if constexpr (remaining) { - std::vector block_values(remaining); -#pragma GCC unroll 8 - for (u32 i = 0; i < remaining; i++) { - block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], scale))); - } - - internal::pack_epi32_fallback(block_values, output); + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + internal::quantize_and_pack_fallback(std::span(input, remaining), scale, remaining, std::span(output, tail_bytes)); } return 0; } diff --git a/include/pernix/x86/bmi2/bmi2_decompression.h b/include/pernix/x86/bmi2/bmi2_decompression.h index cc64ec1..df1d9f8 100644 --- a/include/pernix/x86/bmi2/bmi2_decompression.h +++ b/include/pernix/x86/bmi2/bmi2_decompression.h @@ -226,10 +226,9 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f32 sc } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (u32 i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi32(tail_values[i], scale); - } + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + internal::unpack_and_dequantize_fallback(std::span(input, tail_bytes), remaining, scale, + std::span(output, remaining)); } return 0; @@ -276,10 +275,9 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f64 sc } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (u32 i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi64(tail_values[i], scale); - } + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + internal::unpack_and_dequantize_fallback(std::span(input, tail_bytes), remaining, scale, + std::span(output, remaining)); } return 0; diff --git a/tests/deterministic_correctness_tests.cpp b/tests/deterministic_correctness_tests.cpp index 8345a8c..0d43e56 100644 --- a/tests/deterministic_correctness_tests.cpp +++ b/tests/deterministic_correctness_tests.cpp @@ -280,6 +280,7 @@ TEST(DeterministicCorrectnessTest, SignValuesFalseUsesUnsignedPackedValuesForAll const u32 elements = pernix_elements_per_block(bit_width); std::vector signed_values(elements); std::vector unsigned_values(elements); + std::vector avx512_unsigned_values(elements); ASSERT_EQ( pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, signed_values.data(), true), @@ -289,7 +290,17 @@ TEST(DeterministicCorrectnessTest, SignValuesFalseUsesUnsignedPackedValuesForAll PERNIX_STATUS_OK); const float expected_unsigned = static_cast((1U << bit_width) - 1U); - EXPECT_EQ(unsigned_values[0], expected_unsigned); + EXPECT_TRUE(std::ranges::all_of(unsigned_values, [expected_unsigned](const float value) { return value == expected_unsigned; })); + + const pernix_status avx512_status = pernix_decompress_block_f32(PERNIX_BACKEND_X86_AVX512_VBMI, bit_width, kBlockSize, + packed.data(), 1.0f, avx512_unsigned_values.data(), false); + if (avx512_status == PERNIX_STATUS_OK) { + EXPECT_EQ(avx512_unsigned_values, unsigned_values); + } else { + EXPECT_TRUE(avx512_status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || avx512_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || + avx512_status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + } + if (bit_width == 1) { EXPECT_EQ(signed_values[0], 1.0f); } else { @@ -298,6 +309,38 @@ TEST(DeterministicCorrectnessTest, SignValuesFalseUsesUnsignedPackedValuesForAll } } +TEST(DeterministicCorrectnessTest, Avx512TableDrivenSmallWidthsMatchFallbackForMixedBits) { + constexpr std::array bit_widths{1, 2, 4}; + std::vector packed(kBlockSize); + for (usize i = 0; i < packed.size(); ++i) { + packed[i] = static_cast(i * 37U + 0x5aU); + } + + for (const u8 bit_width : bit_widths) { + SCOPED_TRACE(::testing::Message() << "bit_width=" << static_cast(bit_width)); + const u32 elements = pernix_elements_per_block(bit_width); + + for (const bool sign_values : {false, true}) { + std::vector fallback_output(elements); + std::vector avx512_output(elements); + + ASSERT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, + fallback_output.data(), sign_values), + PERNIX_STATUS_OK); + + const pernix_status status = pernix_decompress_block_f32(PERNIX_BACKEND_X86_AVX512_VBMI, bit_width, kBlockSize, packed.data(), + 1.0f, avx512_output.data(), sign_values); + if (status != PERNIX_STATUS_OK) { + EXPECT_TRUE(status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || status == PERNIX_STATUS_UNSUPPORTED_BACKEND || + status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + GTEST_SKIP(); + } + + EXPECT_EQ(avx512_output, fallback_output); + } + } +} + TEST(DeterministicCorrectnessTest, CppSpanWrappersMatchCApiOutputAndStatus) { constexpr u8 bit_width = 10; const auto input = make_dataset(Pattern::Increasing, bit_width, 1); From deafde6c7eca36563488c139b44bff79d5f1530e Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 16 Jul 2026 15:54:36 +0200 Subject: [PATCH 37/43] feat: introduce AVX2 tail mask functions and optimize packing/unpacking routines --- README.md | 43 ++++++---- examples/cpp/roundtrip.cpp | 10 +-- include/pernix/arm64/neon/tables.h | 86 ++++++++++---------- include/pernix/compat.h | 2 +- include/pernix/x86/avx2/avx2_compression.h | 56 ++++++++----- include/pernix/x86/avx2/avx2_decompression.h | 58 ++++++++----- include/pernix/x86/avx2/avx2_tables.h | 26 +++--- include/pernix/x86/avx512vbmi/tables.h | 2 +- include/pernix/x86/bmi2/bmi2_compression.h | 24 +++++- include/pernix/x86/bmi2/bmi2_decompression.h | 47 +++++++++-- tests/install_consumer/main.cpp | 10 +-- 11 files changed, 228 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index 85fdb82..f95f5bd 100644 --- a/README.md +++ b/README.md @@ -30,11 +30,13 @@ The implementation also accepts explicit block sizes `128`, `256`, `512`, and `1 documented interchange format is the 64-byte block. Packed values are written as a byte stream, least-significant bits first within each value and byte. Compression -zero-fills the output block before packing, so unused tail bits in a block are zero. When `sign_values=true`, decompression +zero-fills the output block before packing, so unused tail bits in a block are zero. When `sign_values=true`, +decompression sign-extends each `N`-bit value before multiplying by scale. When `sign_values=false`, decompression treats the packed value as unsigned. For `bit_width == 1`, signed fallback compression clamps to binary `0/1`. -The scalar fallback packs and unpacks byte-by-byte, so the serialized 64-byte block is not a native integer dump. Backends +The scalar fallback packs and unpacks byte-by-byte, so the serialized 64-byte block is not a native integer dump. +Backends are expected to produce compatible blocks for the same inputs, bit width, scale, and sign mode. ## Quantization And Scale @@ -101,8 +103,10 @@ the lower-level kernels. The `_f32` APIs operate on `float` data and take `float` scale values. The `_f64` APIs operate on `double` data and take `double` scale values. Both use the same packed integer block format for a given bit width. -Given the same backend, inputs, bit width, scale, and sign mode, behavior is deterministic. Different backends are tested -for compatible results, but exact floating-point details should not be treated as a cross-backend ABI guarantee beyond the +Given the same backend, inputs, bit width, scale, and sign mode, behavior is deterministic. Different backends are +tested +for compatible results, but exact floating-point details should not be treated as a cross-backend ABI guarantee beyond +the documented quantization model. ## Backends @@ -122,9 +126,12 @@ Current backend enum values: * `PERNIX_BACKEND_FALLBACK_STDPAR` * `PERNIX_BACKEND_FALLBACK_SIMD` -x86 SIMD kernels are compiled with per-source ISA flags when enabled. Generic dispatch and fallback code are built for the -baseline target. ARM decompression paths exist, but ARM compression is incomplete/stubbed. The stdpar fallback is disabled -by default. When enabled, both the compiled library and the header-only target export stdpar dispatch. If TBB is available, +x86 SIMD kernels are compiled with per-source ISA flags when enabled. Generic dispatch and fallback code are built for +the +baseline target. ARM decompression paths exist, but ARM compression is incomplete/stubbed. The stdpar fallback is +disabled +by default. When enabled, both the compiled library and the header-only target export stdpar dispatch. If TBB is +available, stdpar uses standard parallel execution policies; otherwise it preserves the same 8-value grouping logic while executing sequentially. @@ -141,10 +148,10 @@ sequentially. int main() { constexpr u8 bit_width = 16; constexpr u32 block_size = pernix::compressed_block_size(); - constexpr std::size_t elements = pernix::elements_per_block(bit_width); + constexpr usize elements = pernix::elements_per_block(bit_width); std::array input{}; - for (std::size_t i = 0; i < input.size(); ++i) { + for (usize i = 0; i < input.size(); ++i) { input[i] = std::sin(static_cast(i) * 0.25f); } @@ -221,7 +228,8 @@ cmake -S . -B build -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON ``` The modules in `bindings/fortran/src` bind directly to the C ABI names and currently expose f32 and f64 compression and -decompression entry points. The bindings do not yet install Fortran module files as a packaged Fortran SDK; they are meant +decompression entry points. The bindings do not yet install Fortran module files as a packaged Fortran SDK; they are +meant for in-tree builds and examples. ## Building @@ -243,7 +251,8 @@ Common options: * `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON`: build Fortran bindings and Fortran example/test * `-DPERNIX_ENABLE_X86_AVX2=OFF`, `-DPERNIX_ENABLE_X86_BMI2=OFF`, `-DPERNIX_ENABLE_X86_AVX512VBMI=OFF`: disable specific x86 backends -* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF|AUTO|ON`: control the stdpar fallback backend. TBB is optional and enables parallel standard execution policies when found +* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF|AUTO|ON`: control the stdpar fallback backend. TBB is optional and enables + parallel standard execution policies when found * `-DPERNIX_ENABLE_FALLBACK_SIMD=OFF|AUTO|ON`: control the experimental C++26 `std::simd` fallback backend. `AUTO` enables it when both the compiler and standard library support `std::simd`; `ON` makes missing support a configuration error @@ -284,9 +293,12 @@ If Fortran bindings are enabled: ## Limitations -* Public documentation is centered on fixed 64-byte blocks; larger accepted block sizes are compatibility/internal paths. -* Input values should be finite and within the intended quantization range for portable cross-backend behavior. The scalar - fallback clamps NaN, infinity, and out-of-range scaled values, but that is not yet specified as a cross-backend policy. +* Public documentation is centered on fixed 64-byte blocks; larger accepted block sizes are compatibility/internal + paths. +* Input values should be finite and within the intended quantization range for portable cross-backend behavior. The + scalar + fallback clamps NaN, infinity, and out-of-range scaled values, but that is not yet specified as a cross-backend + policy. * Scale must be positive and finite. * ARM64 compression backends are incomplete/stubbed. * `PERNIX_BACKEND_FALLBACK_STDPAR` is not exported by the compiled library target. @@ -296,7 +308,8 @@ If Fortran bindings are enabled: ## Performance Notes Decompression is the performance-sensitive path. Prefer `PERNIX_BACKEND_AUTO` for normal use so PERNIX can select an -available optimized backend. Use `PERNIX_BACKEND_FALLBACK` when deterministic portable fallback behavior is more important +available optimized backend. Use `PERNIX_BACKEND_FALLBACK` when deterministic portable fallback behavior is more +important than backend selection. A separate benchmark framework exists at . diff --git a/examples/cpp/roundtrip.cpp b/examples/cpp/roundtrip.cpp index 34ffcff..5365937 100644 --- a/examples/cpp/roundtrip.cpp +++ b/examples/cpp/roundtrip.cpp @@ -5,12 +5,12 @@ #include int main() { - constexpr u8 bit_width = 16; - constexpr u32 block_size = pernix::compressed_block_size(); - constexpr std::size_t elements = pernix::elements_per_block(bit_width); + constexpr u8 bit_width = 16; + constexpr u32 block_size = pernix::compressed_block_size(); + constexpr usize elements = pernix::elements_per_block(bit_width); std::array input{}; - for (std::size_t i = 0; i < input.size(); ++i) { + for (usize i = 0; i < input.size(); ++i) { input[i] = std::sin(static_cast(i) * 0.25f); } @@ -35,7 +35,7 @@ int main() { return 2; } - for (std::size_t i = 0; i < input.size(); ++i) { + for (usize i = 0; i < input.size(); ++i) { if (std::abs(restored[i] - input[i]) > scale) { return 3; } diff --git a/include/pernix/arm64/neon/tables.h b/include/pernix/arm64/neon/tables.h index 86dfc6a..32da264 100644 --- a/include/pernix/arm64/neon/tables.h +++ b/include/pernix/arm64/neon/tables.h @@ -9,30 +9,30 @@ namespace pernix::arm64::neon::internal { namespace detail { -inline constexpr std::size_t neon_vector_width = 128; -inline constexpr u8 inactive_lane = 0xff; +inline constexpr usize neon_vector_width = 128; +inline constexpr u8 inactive_lane = 0xff; -template +template constexpr bool table_indices_are_valid(const std::array& table) { return std::ranges::all_of(table, [](const u8 index) { return index == inactive_lane || index < Elements; }); } -template +template constexpr std::array make_primary_permute() { static_assert(LANE_BITS % 8 == 0); - constexpr std::size_t lane_bytes = LANE_BITS / 8; + constexpr usize lane_bytes = LANE_BITS / 8; static_assert(ELEMENTS % lane_bytes == 0); std::array table{}; table.fill(inactive_lane); - for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t first_byte = bit_start / 8; - const std::size_t base = entry * lane_bytes; + for (usize entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const usize bit_start = entry * BIT_WIDTH; + const usize first_byte = bit_start / 8; + const usize base = entry * lane_bytes; - for (std::size_t lane_byte = 0; lane_byte < lane_bytes; ++lane_byte) { + for (usize lane_byte = 0; lane_byte < lane_bytes; ++lane_byte) { table[base + lane_byte] = static_cast(first_byte + lane_byte); } } @@ -40,21 +40,21 @@ constexpr std::array make_primary_permute() { return table; } -template +template constexpr std::array make_spill_permute() { static_assert(LANE_BITS % 8 == 0); - constexpr std::size_t lane_bytes = LANE_BITS / 8; + constexpr usize lane_bytes = LANE_BITS / 8; static_assert(ELEMENTS % lane_bytes == 0); std::array table{}; table.fill(inactive_lane); - for (std::size_t entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t first_byte = bit_start / 8; - const std::size_t bit_offset = bit_start % 8; - const std::size_t base = entry * lane_bytes; + for (usize entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const usize bit_start = entry * BIT_WIDTH; + const usize first_byte = bit_start / 8; + const usize bit_offset = bit_start % 8; + const usize base = entry * lane_bytes; if (bit_offset + BIT_WIDTH > LANE_BITS) { table[base] = static_cast(first_byte + lane_bytes); @@ -64,14 +64,14 @@ constexpr std::array make_spill_permute() { return table; } -template +template constexpr std::array make_shift_right() { std::array table{}; table.fill(0); - for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t bit_offset = bit_start % 8u; + for (usize entry = 0; entry < ELEMENTS; ++entry) { + const usize bit_start = entry * BIT_WIDTH; + const usize bit_offset = bit_start % 8u; table[entry] = -static_cast(bit_offset); } @@ -79,15 +79,15 @@ constexpr std::array make_shift_right() { return table; } -template +template constexpr std::array make_shift_left_for_spill() { std::array table{}; table.fill(0); - for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { - const std::size_t bit_start = entry * BIT_WIDTH; - const std::size_t bit_offset = bit_start % 8u; - const bool spills = bit_offset + BIT_WIDTH > LANE_BITS; + for (usize entry = 0; entry < ELEMENTS; ++entry) { + const usize bit_start = entry * BIT_WIDTH; + const usize bit_offset = bit_start % 8u; + const bool spills = bit_offset + BIT_WIDTH > LANE_BITS; table[entry] = spills ? static_cast(LANE_BITS - bit_offset) : 0; } @@ -95,21 +95,21 @@ constexpr std::array make_shift_left_for_spill() { return table; } -template +template constexpr std::array make_contiguous_permute_32() { static_assert(ELEMENTS % 4 == 0); std::array table{}; table.fill(inactive_lane); - for (std::size_t entry = 0; entry < ELEMENTS / 4; ++entry) { - const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; - const std::size_t bit_end = bit_start + BIT_WIDTH - 1; - const std::size_t first_byte = bit_start / 8; - const std::size_t last_byte = bit_end / 8; - const std::size_t base = entry * 4; + for (usize entry = 0; entry < ELEMENTS / 4; ++entry) { + const usize bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + const usize bit_end = bit_start + BIT_WIDTH - 1; + const usize first_byte = bit_start / 8; + const usize last_byte = bit_end / 8; + const usize base = entry * 4; - for (std::size_t byte = first_byte; byte <= last_byte; ++byte) { + for (usize byte = first_byte; byte <= last_byte; ++byte) { table[base + (byte - first_byte)] = static_cast(byte); } } @@ -117,13 +117,13 @@ constexpr std::array make_contiguous_permute_32() { return table; } -template +template constexpr std::array make_shift_right_32() { std::array table{}; table.fill(0); - for (std::size_t entry = 0; entry < ELEMENTS; ++entry) { - const std::size_t bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + for (usize entry = 0; entry < ELEMENTS; ++entry) { + const usize bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; table[entry] = -static_cast(bit_start % 8u); } @@ -139,8 +139,8 @@ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && VECTOR_WIDTH == detail::neon_vector_width) struct table_unpacking { private: - static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; - static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr usize PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr usize SHIFT_ELEMENTS = VECTOR_WIDTH / 8; public: static constexpr u8 bit_width = BIT_WIDTH; @@ -161,8 +161,8 @@ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && VECTOR_WIDTH == detail::neon_vector_width) struct table_unpacking { private: - static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; - static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 16; + static constexpr usize PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr usize SHIFT_ELEMENTS = VECTOR_WIDTH / 16; public: static constexpr u8 bit_width = BIT_WIDTH; @@ -184,8 +184,8 @@ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && VECTOR_WIDTH == detail::neon_vector_width && START_BIT_OFFSET < 8) struct table_unpacking { private: - static constexpr std::size_t PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; - static constexpr std::size_t SHIFT_ELEMENTS = VECTOR_WIDTH / 32; + static constexpr usize PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr usize SHIFT_ELEMENTS = VECTOR_WIDTH / 32; public: static constexpr u8 bit_width = BIT_WIDTH; diff --git a/include/pernix/compat.h b/include/pernix/compat.h index c81a396..ca3d10e 100644 --- a/include/pernix/compat.h +++ b/include/pernix/compat.h @@ -44,6 +44,6 @@ typedef int64_t i64; typedef float f32; typedef double f64; -typedef size_t usize; +typedef std::size_t usize; #endif // PERNIX_COMPAT_H diff --git a/include/pernix/x86/avx2/avx2_compression.h b/include/pernix/x86/avx2/avx2_compression.h index cb07a0b..377b7f1 100644 --- a/include/pernix/x86/avx2/avx2_compression.h +++ b/include/pernix/x86/avx2/avx2_compression.h @@ -13,6 +13,19 @@ namespace pernix { namespace internal { +template + requires(LANES <= 8) +__always_inline __m256i mm256_tail_mask_epi32() { + return _mm256_setr_epi32(LANES > 0 ? -1 : 0, LANES > 1 ? -1 : 0, LANES > 2 ? -1 : 0, LANES > 3 ? -1 : 0, LANES > 4 ? -1 : 0, + LANES > 5 ? -1 : 0, LANES > 6 ? -1 : 0, LANES > 7 ? -1 : 0); +} + +template + requires(LANES <= 4) +__always_inline __m256i mm256_tail_mask_epi64() { + return _mm256_setr_epi64x(LANES > 0 ? -1 : 0, LANES > 1 ? -1 : 0, LANES > 2 ? -1 : 0, LANES > 3 ? -1 : 0); +} + template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __always_inline __m256i mm256_clamp_signed_epi32(__m256i input) { @@ -458,17 +471,12 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scal } if constexpr (remaining) { - // std::vector block_values(remaining); - // #pragma GCC unroll 8 - // for (u32 i = 0; i < remaining; i++) { - // block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], - // scale))); - // } - // - // internal::pack_epi32_fallback(block_values, output); - constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; - internal::quantize_and_pack_fallback(std::span(input, remaining), scale, remaining, std::span(output, tail_bytes)); + const __m256 source = _mm256_maskload_ps(input, internal::mm256_tail_mask_epi32()); + const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + std::memcpy(output, &packed, tail_bytes); } return 0; @@ -516,17 +524,25 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scal } if constexpr (remaining) { - // std::vector block_values(remaining); - // #pragma GCC unroll 8 - // for (u32 i = 0; i < remaining; i++) { - // block_values[i] = static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], - // scale))); - // } - // - // internal::pack_epi32_fallback(block_values, output); - constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; - internal::quantize_and_pack_fallback(std::span(input, remaining), scale, remaining, std::span(output, tail_bytes)); + constexpr u8 first_lanes = remaining < 4 ? remaining : 4; + constexpr u8 second_lanes = remaining > 4 ? remaining - 4 : 0; + + const __m256d source1 = _mm256_maskload_pd(input, internal::mm256_tail_mask_epi64()); + const __m256d source2 = [&] { + if constexpr (second_lanes > 0) { + return _mm256_maskload_pd(input + 4, internal::mm256_tail_mask_epi64()); + } else { + return _mm256_setzero_pd(); + } + }(); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(combined); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + std::memcpy(output, &packed, tail_bytes); } return 0; diff --git a/include/pernix/x86/avx2/avx2_decompression.h b/include/pernix/x86/avx2/avx2_decompression.h index 94d1fc4..c17fc6a 100644 --- a/include/pernix/x86/avx2/avx2_decompression.h +++ b/include/pernix/x86/avx2/avx2_decompression.h @@ -108,8 +108,8 @@ __m128i mm_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) __m128i mm_unpack_epi32_avx2(const u8* __restrict__ input) { - using unpack_table = unpack_tables_avx2; - constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; + using unpack_table = unpack_tables_avx2; + constexpr usize packed_bytes = (4 * BIT_WIDTH + 7) / 8; __m128i source = _mm_setzero_si128(); std::memcpy(&source, input, packed_bytes); @@ -155,13 +155,8 @@ __m256i mm256_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { */ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -__m256i mm256_unpack_epi32_avx2(const u8* __restrict__ input) { - using unpack_table = unpack_tables_avx2; - constexpr std::size_t packed_bytes = BIT_WIDTH; - - __m256i source = _mm256_setzero_si256(); - std::memcpy(&source, input, packed_bytes); - +__m256i mm256_unpack_epi32_avx2(const __m256i source) { + using unpack_table = unpack_tables_avx2; const __m256i permuted = _mm256_permutevar8x32_epi32(source, unpack_table::get_permute()); const __m256i shuffled = _mm256_shuffle_epi8(permuted, unpack_table::get_shuffle()); @@ -175,6 +170,17 @@ __m256i mm256_unpack_epi32_avx2(const u8* __restrict__ input) { return shifted; } + +/** + * @brief Load and unpack eight values using the table-driven AVX2 shuffle path. + */ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +__m256i mm256_unpack_epi32_avx2(const u8* __restrict__ input) { + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, BIT_WIDTH); + return mm256_unpack_epi32_avx2(source); +} } // namespace internal /** @@ -211,13 +217,13 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f32 sc } if constexpr (remaining > 0) { - // const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - // for (u32 i = 0; i < remaining; i++) { - // output[i] = internal::dequantize_epi32(tail_values[i], scale); - // } constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; - internal::unpack_and_dequantize_fallback(std::span(input, tail_bytes), remaining, scale, - std::span(output, remaining)); + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, tail_bytes); + + const __m256i unpacked = internal::mm256_unpack_epi32_avx2(source); + const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); + std::memcpy(output, &dequantized, remaining * sizeof(f32)); } return 0; @@ -263,14 +269,22 @@ int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f64 sc } if constexpr (remaining > 0) { - // const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - // for (u32 i = 0; i < remaining; i++) { - // output[i] = internal::dequantize_epi64(tail_values[i], scale); - // } - constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; - internal::unpack_and_dequantize_fallback(std::span(input, tail_bytes), remaining, scale, - std::span(output, remaining)); + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, tail_bytes); + + const __m256i unpacked = internal::mm256_unpack_epi32_avx2(source); + const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); + const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); + + const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); + const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); + + constexpr u32 first_elements = remaining < 4 ? remaining : 4; + std::memcpy(output, &dequantized1, first_elements * sizeof(f64)); + if constexpr (remaining > 4) { + std::memcpy(output + 4, &dequantized2, (remaining - 4) * sizeof(f64)); + } } return 0; } diff --git a/include/pernix/x86/avx2/avx2_tables.h b/include/pernix/x86/avx2/avx2_tables.h index 4dd37cb..2ee2643 100644 --- a/include/pernix/x86/avx2/avx2_tables.h +++ b/include/pernix/x86/avx2/avx2_tables.h @@ -721,22 +721,22 @@ struct unpack_tables_avx2 { alignas(32) inline static constexpr std::array shuffle = [] { std::array shuffles{}; shuffles.fill(-1); - constexpr std::size_t rebase_second_half = 4 * ((BIT_WIDTH - 1) / 8); + constexpr usize rebase_second_half = 4 * ((BIT_WIDTH - 1) / 8); - for (std::size_t lane = 0; lane < 2; ++lane) { - for (std::size_t i = 0; i < 4; ++i) { - const std::size_t value_index = lane * 4 + i; + for (usize lane = 0; lane < 2; ++lane) { + for (usize i = 0; i < 4; ++i) { + const usize value_index = lane * 4 + i; - const std::size_t bit_start = value_index * BIT_WIDTH; - const std::size_t byte_start = bit_start / 8; - const std::size_t bit_offset = bit_start % 8; - const std::size_t byte_count = (bit_offset + BIT_WIDTH + 7) / 8; + const usize bit_start = value_index * BIT_WIDTH; + const usize byte_start = bit_start / 8; + const usize bit_offset = bit_start % 8; + const usize byte_count = (bit_offset + BIT_WIDTH + 7) / 8; - const std::size_t rebase = (lane == 0) ? 0 : rebase_second_half; - const std::size_t rel_byte_start = byte_start - rebase; + const usize rebase = (lane == 0) ? 0 : rebase_second_half; + const usize rel_byte_start = byte_start - rebase; - const std::size_t dst = (lane * 4 + i) * 4; - for (std::size_t k = 0; k < byte_count; ++k) { + const usize dst = (lane * 4 + i) * 4; + for (usize k = 0; k < byte_count; ++k) { shuffles[dst + k] = static_cast(rel_byte_start + k); } } @@ -748,7 +748,7 @@ struct unpack_tables_avx2 { alignas(64) inline static constexpr std::array shift = [] { std::array shifts{}; - for (std::size_t lane = 0; lane < 8; ++lane) { + for (usize lane = 0; lane < 8; ++lane) { const int bit_offset = static_cast(lane * BIT_WIDTH); const int bit_in_byte = bit_offset % 8; const int left_shift = 32 - BIT_WIDTH - bit_in_byte; diff --git a/include/pernix/x86/avx512vbmi/tables.h b/include/pernix/x86/avx512vbmi/tables.h index 976c873..2c9ce8e 100644 --- a/include/pernix/x86/avx512vbmi/tables.h +++ b/include/pernix/x86/avx512vbmi/tables.h @@ -9,7 +9,7 @@ #include namespace pernix::internal { -template +template static __always_inline Vec load_table(const std::array& table) { static_assert(sizeof(table) >= sizeof(Vec), "table is smaller than requested SIMD vector"); if constexpr (std::is_same_v) { diff --git a/include/pernix/x86/bmi2/bmi2_compression.h b/include/pernix/x86/bmi2/bmi2_compression.h index 645b273..57e73b7 100644 --- a/include/pernix/x86/bmi2/bmi2_compression.h +++ b/include/pernix/x86/bmi2/bmi2_compression.h @@ -187,7 +187,11 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scal if constexpr (remaining) { constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; - internal::quantize_and_pack_fallback(std::span(input, remaining), scale, remaining, std::span(output, tail_bytes)); + const __m256 source = _mm256_maskload_ps(input, internal::mm256_tail_mask_epi32()); + const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); + const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); + std::memcpy(output, &packed, tail_bytes); } return 0; @@ -233,7 +237,23 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scal if constexpr (remaining) { constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; - internal::quantize_and_pack_fallback(std::span(input, remaining), scale, remaining, std::span(output, tail_bytes)); + constexpr u8 first_lanes = remaining < 4 ? remaining : 4; + constexpr u8 second_lanes = remaining > 4 ? remaining - 4 : 0; + + const __m256d source1 = _mm256_maskload_pd(input, internal::mm256_tail_mask_epi64()); + const __m256d source2 = [&] { + if constexpr (second_lanes > 0) { + return _mm256_maskload_pd(input + 4, internal::mm256_tail_mask_epi64()); + } else { + return _mm256_setzero_pd(); + } + }(); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed = internal::mm256_pack_epi32_bmi2(internal::mm256_clamp_signed_epi32(combined)); + std::memcpy(output, &packed, tail_bytes); } return 0; } diff --git a/include/pernix/x86/bmi2/bmi2_decompression.h b/include/pernix/x86/bmi2/bmi2_decompression.h index df1d9f8..823406a 100644 --- a/include/pernix/x86/bmi2/bmi2_decompression.h +++ b/include/pernix/x86/bmi2/bmi2_decompression.h @@ -62,8 +62,8 @@ __m256i mm256_sign_extend32(__m256i source) { template requires(BIT_WIDTH >= 1 && BIT_WIDTH > 0 && BIT_WIDTH <= 24) __m128i mm_unpack_epi32_bmi2(const u8* __restrict__ input) { - constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; + constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + constexpr usize packed_bytes = (4 * BIT_WIDTH + 7) / 8; __m128i result; if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { @@ -120,9 +120,13 @@ __m128i mm_unpack_epi32_bmi2(const u8* __restrict__ input) { */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__m256i mm256_unpack_epi32_bmi2(const u8* __restrict__ input) { - constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - constexpr std::size_t packed_bytes = BIT_WIDTH; +__m256i mm256_unpack_epi32_bmi2(const __m256i packed_source) { + constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + constexpr usize packed_bytes = BIT_WIDTH; + + alignas(32) u64 packed_words[4]; + _mm256_store_si256(reinterpret_cast<__m256i*>(packed_words), packed_source); + const auto* input = reinterpret_cast(packed_words); __m256i result; if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { @@ -189,6 +193,14 @@ __m256i mm256_unpack_epi32_bmi2(const u8* __restrict__ input) { } return result; } + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__m256i mm256_unpack_epi32_bmi2(const u8* __restrict__ input) { + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, BIT_WIDTH); + return mm256_unpack_epi32_bmi2(source); +} } // namespace internal /** @@ -227,8 +239,12 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f32 sc if constexpr (remaining > 0) { constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; - internal::unpack_and_dequantize_fallback(std::span(input, tail_bytes), remaining, scale, - std::span(output, remaining)); + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, tail_bytes); + + const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(source); + const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); + std::memcpy(output, &dequantized, remaining * sizeof(f32)); } return 0; @@ -276,8 +292,21 @@ int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f64 sc if constexpr (remaining > 0) { constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; - internal::unpack_and_dequantize_fallback(std::span(input, tail_bytes), remaining, scale, - std::span(output, remaining)); + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, tail_bytes); + + const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(source); + const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); + const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); + + const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); + const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); + + constexpr u32 first_elements = remaining < 4 ? remaining : 4; + std::memcpy(output, &dequantized1, first_elements * sizeof(f64)); + if constexpr (remaining > 4) { + std::memcpy(output + 4, &dequantized2, (remaining - 4) * sizeof(f64)); + } } return 0; diff --git a/tests/install_consumer/main.cpp b/tests/install_consumer/main.cpp index 08f70de..edf9ec0 100644 --- a/tests/install_consumer/main.cpp +++ b/tests/install_consumer/main.cpp @@ -4,12 +4,12 @@ #include int main() { - constexpr u8 bit_width = 8; - constexpr u32 block_size = 64; - constexpr std::size_t elements = (block_size * 8U) / bit_width; + constexpr u8 bit_width = 8; + constexpr u32 block_size = 64; + constexpr usize elements = (block_size * 8U) / bit_width; std::array input{}; - for (std::size_t i = 0; i < input.size(); ++i) { + for (usize i = 0; i < input.size(); ++i) { input[i] = static_cast(static_cast(i % 17U) - 8) * 0.125f; } @@ -23,7 +23,7 @@ int main() { return 2; } - for (std::size_t i = 0; i < input.size(); ++i) { + for (usize i = 0; i < input.size(); ++i) { if (std::abs(input[i] - restored[i]) > 0.0f) { return 3; } From 6b28a0fc1a653749e19168140f1916c8616b7d6f Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 16 Jul 2026 20:17:16 +0200 Subject: [PATCH 38/43] feat: implement packing tables and routines for AVX512 and AVX2 with enhanced functionality --- include/pernix/detail/pack_tables.h | 161 +++++++ include/pernix/x86/avx512vbmi/packing.h | 460 +++++++++--------- include/pernix/x86/avx512vbmi/tables.h | 613 +----------------------- 3 files changed, 395 insertions(+), 839 deletions(-) diff --git a/include/pernix/detail/pack_tables.h b/include/pernix/detail/pack_tables.h index 23135ee..e970828 100644 --- a/include/pernix/detail/pack_tables.h +++ b/include/pernix/detail/pack_tables.h @@ -1,4 +1,165 @@ #ifndef PERNIX_PACK_TABLES_H #define PERNIX_PACK_TABLES_H +#include + +#include +#include +#include +#include +#include + +namespace pernix::detail { +namespace internal { +template +constexpr std::array make_primary_permute() { + constexpr usize lane_bits = sizeof(LaneType) * 8; + std::array table{}; + table.fill(static_cast(-1)); + + for (usize output_lane = 0; output_lane < Elements; ++output_lane) { + const usize output_start = output_lane * lane_bits; + const usize output_end = output_start + lane_bits; + usize contributor = 0; + + for (usize input_lane = 0; input_lane < Elements; ++input_lane) { + const usize input_start = input_lane * BitWidth; + const usize input_end = input_start + BitWidth; + + // Half-open intervals overlap when each starts before the other ends. + if (!(input_start < output_end && input_end > output_start)) { + continue; + } + + const auto shift = static_cast(input_start) - static_cast(output_start); + // Nonnegative shifts place input bits into this output lane and are primary contributors. + // Negative shifts cross the output boundary and are emitted by the spill logic instead. + if (shift >= 0 && contributor++ == Contributor) { + table[output_lane] = static_cast(input_lane); + break; + } + } + } + return table; +} + +template +constexpr std::array make_primary_shift() { + constexpr usize lane_bits = sizeof(LaneType) * 8; + const auto permute = make_primary_permute(); + std::array table{}; + table.fill(static_cast(lane_bits)); + + for (usize output_lane = 0; output_lane < Elements; ++output_lane) { + if (permute[output_lane] >= 0) { + table[output_lane] = static_cast(static_cast(permute[output_lane]) * BitWidth - output_lane * lane_bits); + } + } + return table; +} + +template +constexpr std::array make_spill_permute() { + constexpr usize lane_bits = sizeof(LaneType) * 8; + std::array table{}; + table.fill(static_cast(-1)); + + for (usize output_lane = 0; output_lane < Elements; ++output_lane) { + const usize output_start = output_lane * lane_bits; + for (usize input_lane = 0; input_lane < Elements; ++input_lane) { + const usize input_start = input_lane * BitWidth; + const usize input_end = input_start + BitWidth; + if (input_start < output_start && input_end > output_start) { + table[output_lane] = static_cast(input_lane); + break; + } + } + } + return table; +} + +template +constexpr std::array make_spill_shift() { + constexpr usize lane_bits = sizeof(LaneType) * 8; + const auto permute = make_spill_permute(); + std::array table{}; + table.fill(static_cast(lane_bits)); + + for (usize output_lane = 0; output_lane < Elements; ++output_lane) { + if (permute[output_lane] >= 0) { + table[output_lane] = static_cast(output_lane * lane_bits - static_cast(permute[output_lane]) * BitWidth); + } + } + return table; +} + +template +constexpr u64 make_active_mask(const std::array& permute) { + static_assert(Elements <= 64); + u64 mask = 0; + for (usize lane = 0; lane < Elements; ++lane) { + if (permute[lane] >= 0) { + mask |= u64{1} << lane; + } + } + return mask; +} + +template +constexpr std::array, sizeof...(Contributors)> make_primary_permutes(std::index_sequence) { + return {make_primary_permute()...}; +} + +template +constexpr std::array, sizeof...(Contributors)> make_primary_shifts(std::index_sequence) { + return {make_primary_shift()...}; +} + +template +constexpr std::array make_primary_masks(const std::array, Contributors>& permutes) { + std::array masks{}; + for (usize contributor = 0; contributor < Contributors; ++contributor) { + masks[contributor] = make_active_mask(permutes[contributor]); + } + return masks; +} +} // namespace internal + +// Unsupported lane types, bit widths, and vector sizes intentionally fail the focused assertions below. +template +struct pack_table { + static_assert(std::signed_integral, "pack_table LaneType must be a signed integral type"); + static_assert(sizeof(LaneType) == sizeof(u8) || sizeof(LaneType) == sizeof(u16) || sizeof(LaneType) == sizeof(u32), + "pack_table LaneType must use 8-, 16-, or 32-bit lanes"); + static_assert(VectorBytes > 0, "pack_table VectorBytes must be greater than zero"); + static_assert(VectorBytes % sizeof(LaneType) == 0, "pack_table VectorBytes must contain a whole number of lanes"); + + static constexpr usize lane_bits = sizeof(LaneType) * 8; + static constexpr bool valid_bit_width = (sizeof(LaneType) == sizeof(u8) && BitWidth >= 1 && BitWidth <= 8) || + (sizeof(LaneType) == sizeof(u16) && BitWidth >= 9 && BitWidth <= 16) || + (sizeof(LaneType) == sizeof(u32) && BitWidth >= 17 && BitWidth <= 24); + static_assert(valid_bit_width, "pack_table BitWidth is outside the permitted range for LaneType"); + + static constexpr usize element_count = VectorBytes / sizeof(LaneType); + static_assert(element_count <= 64, "pack_table supports at most 64 lanes because active masks use u64"); + static_assert(element_count == 0 || element_count - 1 <= static_cast(std::numeric_limits::max()), + "pack_table permute indices must be representable by LaneType"); + static_assert(std::numeric_limits::lowest() <= -1 && static_cast(-1) < 0, + "pack_table LaneType must represent -1 as the inactive-entry sentinel"); + + static constexpr usize contributor_count = BitWidth > 0 ? (lane_bits + BitWidth - 1) / BitWidth : 0; + static_assert(contributor_count > 0, "pack_table must have at least one primary contributor"); + static constexpr u8 bit_width = BitWidth; + + alignas(64) static constexpr auto primary_permutes = + internal::make_primary_permutes(std::make_index_sequence{}); + alignas(64) static constexpr auto primary_shifts = + internal::make_primary_shifts(std::make_index_sequence{}); + alignas(64) static constexpr auto spill_permute = internal::make_spill_permute(); + alignas(64) static constexpr auto spill_shift = internal::make_spill_shift(); + static constexpr auto primary_masks = internal::make_primary_masks(primary_permutes); + static constexpr u64 spill_mask = internal::make_active_mask(spill_permute); +}; +} // namespace pernix::detail + #endif // PERNIX_PACK_TABLES_H diff --git a/include/pernix/x86/avx512vbmi/packing.h b/include/pernix/x86/avx512vbmi/packing.h index 158d7d1..5424458 100644 --- a/include/pernix/x86/avx512vbmi/packing.h +++ b/include/pernix/x86/avx512vbmi/packing.h @@ -1,46 +1,101 @@ #ifndef PERNIX_AVX512VBMI_PACKING_H #define PERNIX_AVX512VBMI_PACKING_H +#include +#include #include #include +#include +#include + namespace pernix::internal { +template +__always_inline Vec combine_primary_contributors(const PackContributor& pack_contributor, const Combine& combine, + std::index_sequence) { + static_assert(Tables::contributor_count > 0); + static_assert(sizeof...(Indices) + 1 == Tables::contributor_count); + + Vec result = pack_contributor(Tables::primary_permutes[0], Tables::primary_shifts[0], Tables::primary_masks[0]); + ((result = combine(result, pack_contributor(Tables::primary_permutes[Indices + 1], Tables::primary_shifts[Indices + 1], + Tables::primary_masks[Indices + 1]))), + ...); + return result; +} + namespace m128 { +constexpr __mmask16 kPackAlternateByteMask16 = 0xAAAAULL; + +__always_inline __m128i mm_pack_sllv_epi8(const __m128i input, const __m128i count) { + const __m128i high_mask = _mm_set1_epi16(static_cast(0xff00u)); + const __m128i low = _mm_sllv_epi16(input, _mm_andnot_si128(high_mask, count)); + const __m128i high = _mm_sllv_epi16(_mm_and_si128(high_mask, input), _mm_srli_epi16(count, 8)); + return _mm_mask_blend_epi8(kPackAlternateByteMask16, low, high); +} + +__always_inline __m128i mm_pack_srlv_epi8(const __m128i input, const __m128i count) { + const __m128i low_mask = _mm_set1_epi16(0x00ff); + const __m128i low = _mm_srlv_epi16(_mm_and_si128(low_mask, input), _mm_and_si128(low_mask, count)); + const __m128i high = _mm_srlv_epi16(input, _mm_srli_epi16(count, 8)); + return _mm_mask_blend_epi8(kPackAlternateByteMask16, low, high); +} + +template +__always_inline __m128i mm_pack_table_avx512vbmi(const __m128i values) { + using tables = detail::pack_table; + using contributor_table = std::array; + const auto pack_contributor = [values](const contributor_table& permute_indices, const contributor_table& shift_counts, + const u64 active_mask) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m128i value = + _mm_maskz_permutexvar_epi8(static_cast<__mmask16>(active_mask), load_table<__m128i>(permute_indices), values); + return mm_pack_sllv_epi8(value, load_table<__m128i>(shift_counts)); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m128i value = + _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(active_mask), load_table<__m128i>(permute_indices), values); + return _mm_sllv_epi16(value, load_table<__m128i>(shift_counts)); + } else { + const __m128i value = + _mm_maskz_permutexvar_epi32(static_cast<__mmask8>(active_mask), load_table<__m128i>(permute_indices), values); + return _mm_sllv_epi32(value, load_table<__m128i>(shift_counts)); + } + }; + + const auto combine = [](__m128i left, __m128i right) { return _mm_or_si128(left, right); }; + __m128i result = + combine_primary_contributors(pack_contributor, combine, std::make_index_sequence{}); + + if constexpr (tables::spill_mask != 0) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m128i permuted = + _mm_maskz_permutexvar_epi8(static_cast<__mmask16>(tables::spill_mask), load_table<__m128i>(tables::spill_permute), values); + result = _mm_or_si128(result, mm_pack_srlv_epi8(permuted, load_table<__m128i>(tables::spill_shift))); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m128i permuted = + _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(tables::spill_mask), load_table<__m128i>(tables::spill_permute), values); + result = _mm_or_si128(result, _mm_srlv_epi16(permuted, load_table<__m128i>(tables::spill_shift))); + } else { + const __m128i permuted = + _mm_maskz_permutexvar_epi32(static_cast<__mmask8>(tables::spill_mask), load_table<__m128i>(tables::spill_permute), values); + result = _mm_or_si128(result, _mm_srlv_epi32(permuted, load_table<__m128i>(tables::spill_shift))); + } + } + return result; +} + /** * @brief Pack 8 16-bit values for bit widths 9 through 16 using VBMI. */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { +__always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i input) { if constexpr (BIT_WIDTH == 16) { return input; - } else { - using tables = pack_tables_avx512_16; - const __m128i maskv = _mm_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i permuted1 = _mm_permutexvar_epi16(tables::get_permute1(), masked); - const __m128i permuted2 = _mm_permutexvar_epi16(tables::get_permute2(), masked); - - const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm_or_si128(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m128i permuted1 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask1), tables::get_permute1(), masked); - const __m128i permuted2 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask2), tables::get_permute2(), masked); - const __m128i permuted3 = _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(mask3), tables::get_permute3(), masked); - - const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); - } } + + const __m128i maskv = _mm_set1_epi16(static_cast(detail::low_bit_mask())); + const __m128i masked_values = _mm_and_si128(input, maskv); + return mm_pack_table_avx512vbmi(masked_values); } /** @@ -48,44 +103,14 @@ __always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { +__always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i input) { if constexpr (BIT_WIDTH == 8) { return input; - } else { - const __m128i maskv = _mm_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm_set1_epi16(static_cast(_mm_cmpgt_epi8_mask(masked, _mm_setzero_si128()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m128i shifted = _mm_srli_epi16(masked, 6); - const __m128i combined = _mm_or_si128(masked, shifted); - - const __m128i shifted2 = _mm_srli_epi32(combined, 12); - const __m128i combined2 = _mm_or_si128(shifted2, combined); - - return _mm_cvtepi32_epi8(combined2); - } else if constexpr (BIT_WIDTH == 3) { - const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(static_cast(0xFF00u))); - - const __m128i pair6 = _mm_or_si128(even, _mm_srli_epi16(odd, 5)); - const __m128i packed12 = _mm_or_si128(pair6, _mm_srli_epi32(pair6, 10)); - - return m128::mm_pack_epi16_avx512vbmi_9to16<12>(_mm_cvtepi32_epi16(packed12)); - } else if constexpr (BIT_WIDTH == 4) { - const __m128i shifted = _mm_srli_epi16(masked, 4); - const __m128i combined = _mm_or_si128(masked, shifted); - - return _mm_cvtepi16_epi8(combined); - } else { - const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(static_cast(0xFF00u))); - - const __m128i shifted = _mm_or_si128(even, _mm_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } } + + const __m128i maskv = _mm_set1_epi8(static_cast(detail::low_bit_mask())); + const __m128i masked_values = _mm_and_si128(input, maskv); + return mm_pack_table_avx512vbmi(masked_values); } /** @@ -93,60 +118,86 @@ __always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i& input) { - using tables = pack_tables_avx512_24; +__always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i input) { + const __m128i maskv = _mm_set1_epi32(static_cast(detail::low_bit_mask())); + const __m128i masked_values = _mm_and_si128(input, maskv); + return mm_pack_table_avx512vbmi(masked_values); +} +} // namespace m128 - const __m128i maskv = _mm_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); +namespace m256 { +constexpr __mmask32 kPackAlternateByteMask32 = 0xAAAAAAAAULL; - const __m128 permuted1 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute1()); - const __m128 permuted2 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute2()); - const __m128 permuted3 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute3()); +__always_inline __m256i mm256_pack_sllv_epi8(const __m256i input, const __m256i count) { + const __m256i high_mask = _mm256_set1_epi16(static_cast(0xff00u)); + const __m256i low = _mm256_sllv_epi16(input, _mm256_andnot_si256(high_mask, count)); + const __m256i high = _mm256_sllv_epi16(_mm256_and_si256(high_mask, input), _mm256_srli_epi16(count, 8)); + return _mm256_mask_blend_epi8(kPackAlternateByteMask32, low, high); +} - const __m128i shifted1 = _mm_sllv_epi32(_mm_castps_si128(permuted1), tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi32(_mm_castps_si128(permuted2), tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi32(_mm_castps_si128(permuted3), tables::get_shift3()); +__always_inline __m256i mm256_pack_srlv_epi8(const __m256i input, const __m256i count) { + const __m256i low_mask = _mm256_set1_epi16(0x00ff); + const __m256i low = _mm256_srlv_epi16(_mm256_and_si256(low_mask, input), _mm256_and_si256(low_mask, count)); + const __m256i high = _mm256_srlv_epi16(input, _mm256_srli_epi16(count, 8)); + return _mm256_mask_blend_epi8(kPackAlternateByteMask32, low, high); +} - return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); +template +__always_inline __m256i mm256_pack_table_avx512vbmi(const __m256i values) { + using tables = detail::pack_table; + using contributor_table = std::array; + const auto pack_contributor = [values](const contributor_table& permute_indices, const contributor_table& shift_counts, + const u64 active_mask) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m256i value = + _mm256_maskz_permutexvar_epi8(static_cast<__mmask32>(active_mask), load_table<__m256i>(permute_indices), values); + return mm256_pack_sllv_epi8(value, load_table<__m256i>(shift_counts)); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m256i value = + _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(active_mask), load_table<__m256i>(permute_indices), values); + return _mm256_sllv_epi16(value, load_table<__m256i>(shift_counts)); + } else { + const __m256i value = + _mm256_maskz_permutexvar_epi32(static_cast<__mmask8>(active_mask), load_table<__m256i>(permute_indices), values); + return _mm256_sllv_epi32(value, load_table<__m256i>(shift_counts)); + } + }; + + const auto combine = [](__m256i left, __m256i right) { return _mm256_or_si256(left, right); }; + __m256i result = + combine_primary_contributors(pack_contributor, combine, std::make_index_sequence{}); + + if constexpr (tables::spill_mask != 0) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m256i permuted = _mm256_maskz_permutexvar_epi8(static_cast<__mmask32>(tables::spill_mask), + load_table<__m256i>(tables::spill_permute), values); + result = _mm256_or_si256(result, mm256_pack_srlv_epi8(permuted, load_table<__m256i>(tables::spill_shift))); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m256i permuted = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(tables::spill_mask), + load_table<__m256i>(tables::spill_permute), values); + result = _mm256_or_si256(result, _mm256_srlv_epi16(permuted, load_table<__m256i>(tables::spill_shift))); + } else { + const __m256i permuted = _mm256_maskz_permutexvar_epi32(static_cast<__mmask8>(tables::spill_mask), + load_table<__m256i>(tables::spill_permute), values); + result = _mm256_or_si256(result, _mm256_srlv_epi32(permuted, load_table<__m256i>(tables::spill_shift))); + } + } + return result; } -} // namespace m128 -namespace m256 { /** * @brief Pack 16 16-bit values for bit widths 9 through 16 using VBMI. */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) { +__always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i input) { if constexpr (BIT_WIDTH == 16) { return input; - } else { - using tables = pack_tables_avx512_16; - const __m256i maskv = _mm256_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m256i permuted1 = _mm256_permutexvar_epi16(tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_permutexvar_epi16(tables::get_permute2(), masked); - - const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm256_or_si256(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask1), tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask2), tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(mask3), tables::get_permute3(), masked); - - const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); - } } + + const __m256i maskv = _mm256_set1_epi16(static_cast(detail::low_bit_mask())); + const __m256i masked_values = _mm256_and_si256(input, maskv); + return mm256_pack_table_avx512vbmi(masked_values); } /** @@ -154,44 +205,14 @@ __always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { +__always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i input) { if constexpr (BIT_WIDTH == 8) { return input; - } else { - const __m256i maskv = _mm256_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm256_set1_epi32(static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m256i shifted = _mm256_srli_epi16(masked, 6); - const __m256i combined = _mm256_or_si256(masked, shifted); - - const __m256i shifted2 = _mm256_srli_epi32(combined, 12); - const __m256i combined2 = _mm256_or_si256(shifted2, combined); - - return _mm256_castsi128_si256(_mm256_cvtepi32_epi8(combined2)); - } else if constexpr (BIT_WIDTH == 3) { - const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(static_cast(0xFF00u))); - - const __m256i pair6 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 5)); - const __m256i packed12 = _mm256_or_si256(pair6, _mm256_srli_epi32(pair6, 10)); - - return m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm256_castsi128_si256(_mm256_cvtepi32_epi16(packed12))); - } else if constexpr (BIT_WIDTH == 4) { - const __m256i shifted = _mm256_srli_epi16(masked, 4); - const __m256i combined = _mm256_or_si256(masked, shifted); - - return _mm256_castsi128_si256(_mm256_cvtepi16_epi8(combined)); - } else { - const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(static_cast(0xFF00u))); - - const __m256i shifted = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm256_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } } + + const __m256i maskv = _mm256_set1_epi8(static_cast(detail::low_bit_mask())); + const __m256i masked_values = _mm256_and_si256(input, maskv); + return mm256_pack_table_avx512vbmi(masked_values); } /** @@ -199,60 +220,86 @@ __always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i& input) { - using tables = pack_tables_avx512_24; +__always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i input) { + const __m256i maskv = _mm256_set1_epi32(static_cast(detail::low_bit_mask())); + const __m256i masked_values = _mm256_and_si256(input, maskv); + return mm256_pack_table_avx512vbmi(masked_values); +} +} // namespace m256 - const __m256i maskv = _mm256_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); +namespace m512 { +constexpr __mmask64 kPackAlternateByteMask64 = 0xAAAAAAAAAAAAAAAAULL; - const __m256i permuted1 = _mm256_permutexvar_epi32(tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_permutexvar_epi32(tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_permutexvar_epi32(tables::get_permute3(), masked); +__always_inline __m512i mm512_pack_sllv_epi8(const __m512i input, const __m512i count) { + const __m512i high_mask = _mm512_set1_epi16(static_cast(0xff00u)); + const __m512i low = _mm512_sllv_epi16(input, _mm512_andnot_si512(high_mask, count)); + const __m512i high = _mm512_sllv_epi16(_mm512_and_si512(high_mask, input), _mm512_srli_epi16(count, 8)); + return _mm512_mask_blend_epi8(kPackAlternateByteMask64, low, high); +} - const __m256i shifted1 = _mm256_sllv_epi32(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi32(permuted2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi32(permuted3, tables::get_shift3()); +__always_inline __m512i mm512_pack_srlv_epi8(const __m512i input, const __m512i count) { + const __m512i low_mask = _mm512_set1_epi16(0x00ff); + const __m512i low = _mm512_srlv_epi16(_mm512_and_si512(low_mask, input), _mm512_and_si512(low_mask, count)); + const __m512i high = _mm512_srlv_epi16(input, _mm512_srli_epi16(count, 8)); + return _mm512_mask_blend_epi8(kPackAlternateByteMask64, low, high); +} - return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); +template +__always_inline __m512i mm512_pack_table_avx512vbmi(const __m512i values) { + using tables = detail::pack_table; + using contributor_table = std::array; + const auto pack_contributor = [values](const contributor_table& permute_indices, const contributor_table& shift_counts, + const u64 active_mask) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m512i value = + _mm512_maskz_permutexvar_epi8(static_cast<__mmask64>(active_mask), load_table<__m512i>(permute_indices), values); + return mm512_pack_sllv_epi8(value, load_table<__m512i>(shift_counts)); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m512i value = + _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(active_mask), load_table<__m512i>(permute_indices), values); + return _mm512_sllv_epi16(value, load_table<__m512i>(shift_counts)); + } else { + const __m512i value = + _mm512_maskz_permutexvar_epi32(static_cast<__mmask16>(active_mask), load_table<__m512i>(permute_indices), values); + return _mm512_sllv_epi32(value, load_table<__m512i>(shift_counts)); + } + }; + + const auto combine = [](__m512i left, __m512i right) { return _mm512_or_si512(left, right); }; + __m512i result = + combine_primary_contributors(pack_contributor, combine, std::make_index_sequence{}); + + if constexpr (tables::spill_mask != 0) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m512i permuted = _mm512_maskz_permutexvar_epi8(static_cast<__mmask64>(tables::spill_mask), + load_table<__m512i>(tables::spill_permute), values); + result = _mm512_or_si512(result, mm512_pack_srlv_epi8(permuted, load_table<__m512i>(tables::spill_shift))); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m512i permuted = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(tables::spill_mask), + load_table<__m512i>(tables::spill_permute), values); + result = _mm512_or_si512(result, _mm512_srlv_epi16(permuted, load_table<__m512i>(tables::spill_shift))); + } else { + const __m512i permuted = _mm512_maskz_permutexvar_epi32(static_cast<__mmask16>(tables::spill_mask), + load_table<__m512i>(tables::spill_permute), values); + result = _mm512_or_si512(result, _mm512_srlv_epi32(permuted, load_table<__m512i>(tables::spill_shift))); + } + } + return result; } -} // namespace m256 -namespace m512 { /** * @brief Pack 32 16-bit values for bit widths 9 through 16 using VBMI. */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) { +__always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i input) { if constexpr (BIT_WIDTH == 16) { return input; - } else { - using tables = pack_tables_avx512_16; - const __m512i maskv = _mm512_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m512i permuted1 = _mm512_permutexvar_epi16(tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_permutexvar_epi16(tables::get_permute2(), masked); - - const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm512_or_si512(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask1), tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask2), tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(mask3), tables::get_permute3(), masked); - - const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); - const __m512i shifted3 = _mm512_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); - } } + + const __m512i maskv = _mm512_set1_epi16(static_cast(detail::low_bit_mask())); + const __m512i masked_values = _mm512_and_si512(input, maskv); + return mm512_pack_table_avx512vbmi(masked_values); } /** @@ -260,44 +307,14 @@ __always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { +__always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i input) { if constexpr (BIT_WIDTH == 8) { return input; - } else { - const __m512i maskv = _mm512_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm512_set1_epi64(static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m512i shifted = _mm512_srli_epi16(masked, 6); - const __m512i combined = _mm512_or_si512(masked, shifted); - - const __m512i shifted2 = _mm512_srli_epi32(combined, 12); - const __m512i combined2 = _mm512_or_si512(shifted2, combined); - - return _mm512_castsi128_si512(_mm512_cvtepi32_epi8(combined2)); - } else if constexpr (BIT_WIDTH == 3) { - const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(static_cast(0xFF00u))); - - const __m512i pair6 = _mm512_or_si512(even, _mm512_srli_epi16(odd, 5)); - const __m512i packed12 = _mm512_or_si512(pair6, _mm512_srli_epi32(pair6, 10)); - - return _mm512_castsi256_si512(m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm512_cvtepi32_epi16(packed12))); - } else if constexpr (BIT_WIDTH == 4) { - const __m512i shifted = _mm512_srli_epi16(masked, 4); - const __m512i combined = _mm512_or_si512(masked, shifted); - - return _mm512_castsi256_si512(_mm512_cvtepi16_epi8(combined)); - } else { - const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(static_cast(0xFF00u))); - - const __m512i shifted = _mm512_or_si512(even, _mm512_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm512_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } } + + const __m512i maskv = _mm512_set1_epi8(static_cast(detail::low_bit_mask())); + const __m512i masked_values = _mm512_and_si512(input, maskv); + return mm512_pack_table_avx512vbmi(masked_values); } /** @@ -305,21 +322,10 @@ __always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i& input) { - using tables = pack_tables_avx512_24; - - const __m512i maskv = _mm512_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - const __m512i permuted1 = _mm512_permutexvar_epi32(tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_permutexvar_epi32(tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_permutexvar_epi32(tables::get_permute3(), masked); - - const __m512i shifted1 = _mm512_sllv_epi32(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi32(permuted2, tables::get_shift2()); - const __m512i shifted3 = _mm512_srlv_epi32(permuted3, tables::get_shift3()); - - return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); +__always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i input) { + const __m512i maskv = _mm512_set1_epi32(static_cast(detail::low_bit_mask())); + const __m512i masked_values = _mm512_and_si512(input, maskv); + return mm512_pack_table_avx512vbmi(masked_values); } } // namespace m512 } // namespace pernix::internal diff --git a/include/pernix/x86/avx512vbmi/tables.h b/include/pernix/x86/avx512vbmi/tables.h index 2c9ce8e..045a93a 100644 --- a/include/pernix/x86/avx512vbmi/tables.h +++ b/include/pernix/x86/avx512vbmi/tables.h @@ -3,10 +3,8 @@ #include -#include #include -#include -#include +#include namespace pernix::internal { template @@ -20,615 +18,6 @@ static __always_inline Vec load_table(const std::array& table) { return _mm_load_si128(reinterpret_cast(table.data())); } } - -template - requires(N >= 9 && N <= 15) -struct pack_tables_avx512_16 { - alignas(64) inline static constexpr std::array permute1 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 0, 2, 4, 6, - -1, 9, 11, 13, - 15, 16, 18, 20, - 22, -1, 25, 27, - - 29, 31, 32, 34, - 36, 38, -1, 41, - 43, 45, 47, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 0, 2, -1, 5, - 7, 8, 10, -1, - 13, 15, 16, 18, - -1, 21, 23, 24, - - 26, -1, 29, 31, - 32, 34, -1, 37, - 39, 40, 42, -1, - 45, 47, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 0, 2, 3, 5, - 6, -1, 9, -1, - 12, -1, 15, 16, - 18, 19, 21, 22, - - -1, 25, -1, 28, - -1, 31, 32, 34, - 35, 37, 38, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 12) { - return std::array{ - 1, 2, 3, 5, - 6, 7, 9, 10, - 11, 13, 14, 15, - 17, 18, 19, 21, - - 22, 23, 25, 26, - 27, 29, 30, 31, - 33, 34, 35, 37, - 38, 39, 41, 42 - }; - } else if constexpr (N == 13) { - return std::array{ - 0, -1, 3, 4, - 5, -1, -1, 9, - 10, -1, -1, 14, - 15, 16, -1, 19, - - 20, 21, -1, -1, - 25, 26, -1, -1, - 30, 31, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 14) { - return std::array{ - 1, 2, 3, 4, - 5, 6, 7, 9, - 10, 11, 12, 13, - 14, 15, 17, 18, - - 19, 20, 21, 22, - 23, 25, 26, 27, - 28, 29, 30, 31, - -1, -1, -1, -1 - }; - } else if constexpr (N == 15) { - return std::array{ - 1, 2, 3, 4, - 5, 6, 7, 8, - 9, 10, 11, 12, - 13, 14, 15, 17, - - 18, 19, 20, 21, - 22, 23, 24, 25, - 26, 27, 28, 29, - 30, 31, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - alignas(64) inline static constexpr std::array permute2 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 1, 3, 5, 7, - 8, 10, 12, 14, - -1, 17, 19, 21, - 23, 24, 26, 28, - - 30, -1, 33, 35, - 37, 39, 40, 42, - 44, 46, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 1, 3, 4, 6, - -1, 9, 11, 12, - 14, -1, 17, 19, - 20, 22, -1, 25, - - 27, 28, 30, -1, - 33, 35, 36, 38, - -1, 41, 43, 44, - 46, -1, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 1, -1, 4, -1, - 7, 8, 10, 11, - 13, 14, -1, 17, - -1, 20, -1, 23, - - 24, 26, 27, 29, - 30, -1, 33, -1, - 36, -1, 39, 40, - -1, -1, -1, -1 - }; - } else if constexpr (N == 12) { - return std::array{ - 0, 1, 2, 4, - 5, 6, 8, 9, - 10, 12, 13, 14, - 16, 17, 18, 20, - - 21, 22, 24, 25, - 26, 28, 29, 30, - 32, 33, 34, 36, - 37, 38, 40, 41 - }; - } else if constexpr (N == 13) { - return std::array{ - 1, 2, -1, -1, - 6, 7, 8, -1, - 11, 12, 13, -1, - -1, 17, 18, -1, - - -1, 22, 23, 24, - -1, 27, 28, 29, - -1, -1, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 14) { - return std::array{ - 0, 1, 2, 3, - 4, 5, 6, 8, - 9, 10, 11, 12, - 13, 14, 16, 17, - - 18, 19, 20, 21, - 22, 24, 25, 26, - 27, 28, 29, 30, - -1, -1, -1, -1 - }; - } else if constexpr (N == 15) { - return std::array{ - 0, 1, 2, 3, - 4, 5, 6, 7, - 8, 9, 10, 11, - 12, 13, 14, 16, - - 17, 18, 19, 20, - 21, 22, 23, 24, - 25, 26, 27, 28, - 29, 30, -1, -1 - }; - } - // clang-format on - }(); - - alignas(64) inline static constexpr std::array permute3 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - -1, 1, 3, 5, - 7, 8, 10, 12, - 14, -1, 17, 19, - 21, 23, 24, 26, - - 28, 30, -1, 33, - 35, 37, 39, 40, - 42, 44, 46, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - -1, 1, 3, 4, - 6, -1, 9, 11, - 12, 14, -1, 17, - 19, 20, 22, -1, - - 25, 27, 28, 30, - -1, 33, 35, 36, - 38, -1, 41, 43, - 44, 46, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - -1, 1, 2, 4, - 5, 7, 8, 10, - 11, 13, 14, -1, - 17, 18, 20, 21, - - 23, 24, 26, 27, - 29, 30, -1, 33, - 34, 36, 37, 39, - -1, -1, -1, -1 - }; - } else if constexpr (N == 13) { - return std::array{ - -1, 1, 2, 3, - 4, 6, 7, 8, - 9, 11, 12, 13, - 14, -1, 17, 18, - - 19, 20, 22, 23, - 24, 25, 27, 28, - 29, 30, -1, -1, - -1, -1, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - alignas(64) inline static constexpr std::array shift1 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 0, 2, 4, 6, - 0, 1, 3, 5, - 7, 0, 2, 4, - - 6, 0, 1, 3, - 5, 7, 0, 2, - 4, 6, 0, 1, - 3, 5, 7, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 0, 4, 0, 2, - 6, 0, 4, 0, - 2, 6, 0, 4, - 0, 2, 6, 0, - - 4, 0, 2, 6, - 0, 4, 0, 2, - 6, 0, 4, 0, - 2, 6, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 0, 6, 1, 7, - 2, 0, 3, 0, - 4, 0, 5, 0, - 6, 1, 7, 2, - - 0, 3, 0, 4, - 0, 5, 0, 6, - 1, 7, 2, 0, - -1, -1, -1, -1 - }; - } else if constexpr (N == 12) { - return std::array{ - 12, 8, 4, 12, - 8, 4, 12, 8, - 4, 12, 8, 4, - 12, 8, 4, 12, - - 8, 4, 12, 8, - 4, 12, 8, 4, - 12, 8, 4, 12, - 8, 4, 12, 8 - }; - } else if constexpr (N == 13) { - return std::array{ - 0, 0, 7, 4, - 1, 0, 0, 5, - 2, 0, 0, 6, - 3, 0, 0, 7, - - 4, 1, 0, 0, - 5, 2, 0, 0, - 6, 3, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 14) { - return std::array{ - 14, 12, 10, 8, - 6, 4, 2, 14, - 12, 10, 8, 6, - 4, 2, 14, 12, - - 10, 8, 6, 4, - 2, 14, 12, 10, - 8, 6, 4, 2, - -1, -1, -1, -1 - }; - } else if constexpr (N == 15) { - return std::array{ - 15, 14, 13, 12, - 11, 10, 9, 8, - 7, 6, 5, 4, - 3, 2, 1, 15, - - 14, 13, 12, 11, - 10, 9, 8, 7, - 6, 5, 4, 3, - 2, 1, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - alignas(64) inline static constexpr std::array shift2 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 9, 11, 13, 15, - 8, 10, 12, 14, - 8, 9, 11, 13, - 15, 8, 10, 12, - - 14, 8, 9, 11, - 13, 15, 8, 10, - 12, 14, 8, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 10, 14, 8, 12, - 0, 10, 14, 8, - 12, 0, 10, 14, - 8, 12, 0, 10, - - 14, 8, 12, 0, - 10, 14, 8, 12, - 0, 10, 14, 8, - 12, 0, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 11, 8, 12, 8, - 13, 8, 14, 9, - 15, 10, 8, 11, - - 8, 12, 8, 13, - 8, 14, 9, 15, - 10, 8, 11, 8, - 12, 8, 13, 8, - -1, -1, -1, -1 - }; - } else if constexpr (N == 12) { - return std::array{ - 0, 4, 8, 0, - 4, 8, 0, 4, - 8, 0, 4, 8, - 0, 4, 8, 0, - - 4, 8, 0, 4, - 8, 0, 4, 8, - 0, 4, 8, 0, - 4, 8, 0, 4 - }; - } else if constexpr (N == 13) { - return std::array{ - 13, 10, 0, 0, - 14, 11, 8, 0, - 15, 12, 9, 0, - 0, 13, 10, 0, - - 0, 14, 11, 8, - 0, 15, 12, 9, - 0, 0, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 14) { - return std::array{ - 0, 2, 4, 6, - 8, 10, 12, 0, - 2, 4, 6, 8, - 10, 12, 0, 2, - - 4, 6, 8, 10, - 12, 0, 2, 4, - 6, 8, 10, 12, - -1, -1, -1, -1 - }; - } else if constexpr (N == 15) { - return std::array{ - 0, 1, 2, 3, - 4, 5, 6, 7, - 8, 9, 10, 11, - 12, 13, 14, 0, - - 1, 2, 3, 4, - 5, 6, 7, 8, - 9, 10, 11, 12, - 13, 14, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - alignas(64) inline static constexpr std::array shift3 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 0, 7, 5, 3, - 1, 8, 6, 4, - 2, 0, 7, 5, - 3, 1, 8, 6, - - 4, 2, 0, 7, - 5, 3, 1, 8, - 6, 4, 2, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 0, 6, 2, 8, - 4, 0, 6, 2, - 8, 4, 0, 6, - 2, 8, 4, 0, - - 6, 2, 8, 4, - 0, 6, 2, 8, - 4, 0, 6, 2, - 8, 4, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 0, 5, 10, 4, - 9, 3, 8, 2, - 7, 1, 6, 0, - 5, 10, 4, 9, - - 3, 8, 2, 7, - 1, 6, 0, 5, - 10, 4, 9, 3, - -1, -1, -1, -1 - }; - } else if constexpr (N == 13) { - return std::array{ - 0, 3, 6, 9, - 12, 2, 5, 8, - 11, 1, 4, 7, - 10, 0, 3, 6, - - 9, 12, 2, 5, - 8, 11, 1, 4, - 7, 10, -1, -1, - -1, -1, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - inline static constexpr std::tuple<__mmask32, __mmask32, __mmask32> get_permute_masks() { - // clang-format off - if constexpr (N == 9) { - return { - 0x07BFDFEF, - 0x03FDFEFF, - 0x07FBFDFE - }; - } else if constexpr (N == 10) { - return { - 0x37BDEF7B, - 0x1EF7BDEF, - 0x3DEF7BDE - }; - } else if constexpr (N == 11) { - return { - 0x07EAFD5F, - 0x0D5FABF5, - 0x0FBFF7FE - }; - } else if constexpr (N == 13) { - return { - 0x0333B99D, - 0x00EE6773, - 0x03FFDFFE - }; - } - return {0, 0, 0}; - // clang-format on - } - - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } - static __always_inline Vec get_permute3() { return load_table(permute3); } - - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } - static __always_inline Vec get_shift3() { return load_table(shift3); } -}; - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct pack_tables_avx512_24 { -private: - struct word_plan { - i32 left_index1 = -1; - i32 left_index2 = -1; - i32 right_index = -1; - u32 left_shift1 = 32; - u32 left_shift2 = 32; - u32 right_shift = 32; - }; - - static constexpr word_plan create_plan(const u32 idx) { - word_plan plan{}; - - const u32 word_start = idx * 32u; - const u32 word_end = word_start + 32u; - - u32 left_slot = 0; - for (u32 input_lane = 0; input_lane < 16; ++input_lane) { - const u32 input_start = input_lane * BIT_WIDTH; - const u32 input_end = input_start + BIT_WIDTH; - - const u32 overlap_start = std::max(word_start, input_start); - const u32 overlap_end = std::min(word_end, input_end); - if (overlap_start >= overlap_end) { - continue; - } - - const auto output_bit = static_cast(overlap_start - word_start); - const auto input_bit = static_cast(overlap_start - input_start); - const i32 delta = output_bit - input_bit; - - if (delta >= 0) { - if (left_slot == 0) { - plan.left_index1 = static_cast(input_lane); - plan.left_shift1 = static_cast(delta); - ++left_slot; - } else { - plan.left_index2 = static_cast(input_lane); - plan.left_shift2 = static_cast(delta); - } - } else { - plan.right_index = static_cast(input_lane); - plan.right_shift = static_cast(-delta); - } - } - - return plan; - } - - static constexpr std::array word_plans = [] { - std::array plans{}; - for (u32 i = 0; i < 16; ++i) { - plans[i] = create_plan(i); - } - return plans; - }(); - - template - static __always_inline constexpr std::array make_table(Getter getter) { - std::array values{}; - for (u32 i = 0; i < 16; ++i) { - values[i] = getter(word_plans[i]); - } - return values; - } - - alignas(64) static constexpr auto permute1 = make_table([](const word_plan& p) { return p.left_index1; }); - - alignas(64) static constexpr auto permute2 = make_table([](const word_plan& p) { return p.left_index2; }); - - alignas(64) static constexpr auto permute3 = make_table([](const word_plan& p) { return p.right_index; }); - - alignas(64) static constexpr auto shift1 = make_table([](const word_plan& p) { return p.left_shift1; }); - - alignas(64) static constexpr auto shift2 = make_table([](const word_plan& p) { return p.left_shift2; }); - - alignas(64) static constexpr auto shift3 = make_table([](const word_plan& p) { return p.right_shift; }); - -public: - static __always_inline Vec get_permute1() { return load_table(permute1); } - static __always_inline Vec get_permute2() { return load_table(permute2); } - static __always_inline Vec get_permute3() { return load_table(permute3); } - - static __always_inline Vec get_shift1() { return load_table(shift1); } - static __always_inline Vec get_shift2() { return load_table(shift2); } - static __always_inline Vec get_shift3() { return load_table(shift3); } -}; - } // namespace pernix::internal #endif // PERNIX_AVX512VBMI_TABLES_H From ffba31260e5f67af2175810964009a9e76c97bb2 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 16 Jul 2026 20:37:01 +0200 Subject: [PATCH 39/43] refactor: remove reference qualifiers from function parameters for improved consistency --- include/pernix/arm64/neon/common.h | 20 ++++----- include/pernix/arm64/neon/unpacking.h | 6 +-- include/pernix/x86/avx2/avx2_compression.h | 42 +++++++++---------- include/pernix/x86/avx2/avx2_decompression.h | 8 ++-- .../x86/avx512vbmi/avx512vbmi_compression.h | 6 +-- .../x86/avx512vbmi/avx512vbmi_decompression.h | 4 +- include/pernix/x86/avx512vbmi/unpacking.h | 18 ++++---- include/pernix/x86/bmi2/bmi2_compression.h | 10 ++--- 8 files changed, 57 insertions(+), 57 deletions(-) diff --git a/include/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h index dcfd664..be0d23e 100644 --- a/include/pernix/arm64/neon/common.h +++ b/include/pernix/arm64/neon/common.h @@ -16,7 +16,7 @@ static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements return tail_bytes; } -__always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(const int8x16_t& input) { +__always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(int8x16_t input) { const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); @@ -28,14 +28,14 @@ __always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(const int8x16_t& inpu }}; } -__always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(const int16x8_t& input) { +__always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(int16x8_t input) { return {{ vmovl_s16(vget_low_s16(input)), vmovl_s16(vget_high_s16(input)), }}; } -__always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, const float32x4_t& scale) { +__always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, float32x4_t scale) { return {{ vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), @@ -44,29 +44,29 @@ __always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, co }}; } -__always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t& input, const float32x4_t& scale) { +__always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t& input, float32x4_t scale) { return {{ vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), }}; } -__always_inline float32x4_t neon_dequantize_epi32(const int32x4_t& input, const float32x4_t& scale) { +__always_inline float32x4_t neon_dequantize_epi32(int32x4_t input, float32x4_t scale) { return vmulq_f32(vcvtq_f32_s32(input), scale); } -__always_inline float64x2_t neon_dequantize_epi32_f64(const int32x2_t& input, const float64x2_t& scale) { +__always_inline float64x2_t neon_dequantize_epi32_f64(int32x2_t input, float64x2_t scale) { return vmulq_f64(vcvtq_f64_s64(vmovl_s32(input)), scale); } -__always_inline float64x2x2_t neon_dequantize_epi32_f64(const int32x4_t& input, const float64x2_t& scale) { +__always_inline float64x2x2_t neon_dequantize_epi32_f64(int32x4_t input, float64x2_t scale) { return {{ neon_dequantize_epi32_f64(vget_low_s32(input), scale), neon_dequantize_epi32_f64(vget_high_s32(input), scale), }}; } -__always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t& input, const float64x2_t& scale) { +__always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t& input, float64x2_t scale) { const float64x2x2_t dequantized_low = neon_dequantize_epi32_f64(input.val[0], scale); const float64x2x2_t dequantized_high = neon_dequantize_epi32_f64(input.val[1], scale); @@ -78,7 +78,7 @@ __always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t& input }}; } -__always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input, const float64x2_t& scale) { +__always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input, float64x2_t scale) { const float64x2x2_t dequantized0 = neon_dequantize_epi32_f64(input.val[0], scale); const float64x2x2_t dequantized1 = neon_dequantize_epi32_f64(input.val[1], scale); const float64x2x2_t dequantized2 = neon_dequantize_epi32_f64(input.val[2], scale); @@ -166,7 +166,7 @@ __always_inline void neon_store_tail_elements_f32(float32_t* output, const float std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); } -__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4_t& data, const u32 tail_elements) { +__always_inline void neon_store_tail_elements_f32(float32_t* output, float32x4_t data, const u32 tail_elements) { float32_t buffer[4]; vst1q_f32(buffer, data); std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h index 2648676..93b8994 100644 --- a/include/pernix/arm64/neon/unpacking.h +++ b/include/pernix/arm64/neon/unpacking.h @@ -9,7 +9,7 @@ using namespace pernix::arm64::neon::internal; namespace pernix::arm64::neon::internal::b128 { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t& input) { +__always_inline int8x16_t neon_unpack_epi8_1to8(uint8x16_t input) { if constexpr (BIT_WIDTH == 8) { return vreinterpretq_s8_u8(input); } else if constexpr (BIT_WIDTH == 1) { @@ -45,7 +45,7 @@ __always_inline int8x16_t neon_unpack_epi8_1to8(const uint8x16_t& input) { template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t& input) { +__always_inline int16x8_t neon_unpack_epi16_9to16(uint16x8_t input) { if constexpr (BIT_WIDTH == 16) { return vreinterpretq_s16_u16(input); } else { @@ -78,7 +78,7 @@ __always_inline int16x8_t neon_unpack_epi16_9to16(const uint16x8_t& input) { template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline int32x4_t neon_unpack_epi32_17to24(const uint32x4_t& input) { +__always_inline int32x4_t neon_unpack_epi32_17to24(uint32x4_t input) { using tables = table_unpacking; const uint8x16_t input_8 = vreinterpretq_u8_u32(input); diff --git a/include/pernix/x86/avx2/avx2_compression.h b/include/pernix/x86/avx2/avx2_compression.h index 377b7f1..efc4a4e 100644 --- a/include/pernix/x86/avx2/avx2_compression.h +++ b/include/pernix/x86/avx2/avx2_compression.h @@ -41,7 +41,7 @@ __always_inline __m256i mm256_clamp_signed_epi32(__m256i input) { * @param scale per-lane scale factor. * @return __m128i quantized values. */ -__always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& scale) { +__always_inline __m128i mm_quantize_ps_epi32(__m128 input, __m128 scale) { const __m128 scaled = _mm_mul_ps(input, scale); // const __m128 rounded = _mm_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); return _mm_cvtps_epi32(scaled); @@ -54,7 +54,7 @@ __always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& * @param scale per-lane scale factor. * @return __m128i quantized values in the low lanes. */ -__always_inline __m128i mm_quantize_pd_epi32(const __m128d& input, const __m128d& scale) { +__always_inline __m128i mm_quantize_pd_epi32(__m128d input, __m128d scale) { const __m128d scaled = _mm_mul_pd(input, scale); return _mm_cvtpd_epi32(scaled); } @@ -66,7 +66,7 @@ __always_inline __m128i mm_quantize_pd_epi32(const __m128d& input, const __m128d * @param scale per-lane scale factor. * @return __m256i quantized values. */ -__always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m256& scale) { +__always_inline __m256i mm256_quantize_ps_epi32(__m256 input, __m256 scale) { const __m256 scaled = _mm256_mul_ps(input, scale); // const __m256 rounded = _mm256_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); return _mm256_cvtps_epi32(scaled); @@ -79,7 +79,7 @@ __always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m25 * @param scale per-lane scale factor. * @return __m128i quantized values in the low lanes. */ -__always_inline __m128i mm256_quantize_pd_epi32(const __m256d& input, const __m256d& scale) { +__always_inline __m128i mm256_quantize_pd_epi32(__m256d input, __m256d scale) { const __m256d scaled = _mm256_mul_pd(input, scale); return _mm256_cvtpd_epi32(scaled); } @@ -189,7 +189,7 @@ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i c */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) -__always_inline auto mm_pack_epi32_avx2_1to3(__m128i& input) -> __m128i { +__always_inline auto mm_pack_epi32_avx2_1to3(__m128i input) -> __m128i { constexpr u32 bitmask = (1U << BIT_WIDTH) - 1U; const __m128i masked = _mm_and_si128(input, _mm_set1_epi32(static_cast(bitmask))); @@ -207,7 +207,7 @@ __always_inline auto mm_pack_epi32_avx2_1to3(__m128i& input) -> __m128i { */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) -__always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { +__always_inline __m256i mm256_pack_epi32_avx2_1to3(__m256i input) { constexpr u32 bitmask = (1u << BIT_WIDTH) - 1u; const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(static_cast(bitmask))); @@ -225,7 +225,7 @@ __always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { return _mm256_castsi128_si256(x); } -__always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i& input) { +__always_inline __m256i mm256_pack_epi32_avx2_4(__m256i input) { const __m256i zero = _mm256_setzero_si256(); const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(0x0f)); @@ -246,7 +246,7 @@ __always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i& input) { */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { +__always_inline auto mm_pack_epi32_avx2_9to16(__m128i input) -> __m128i { using tables = pack_tables_avx2_16; constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; const __m128i masked = _mm_and_si128(input, _mm_set1_epi16(bitmask)); @@ -279,7 +279,7 @@ __always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { */ template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i& input) -> __m256i { +__always_inline auto mm256_pack_epi32_avx2_9to16(__m256i input) -> __m256i { using tables = pack_tables_avx2_16; constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; const __m128i packed = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); @@ -310,7 +310,7 @@ __always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i& input) -> __m256 template requires(BIT_WIDTH >= 5 && BIT_WIDTH <= 7) -__always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i { +__always_inline auto mm256_pack_epi32_avx2_5to7(__m256i input) -> __m256i { const __m256i zero = _mm256_setzero_si256(); const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32((1 << BIT_WIDTH) - 1)); @@ -332,7 +332,7 @@ __always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i */ template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m256i { +__always_inline auto mm256_pack_epi32_avx2_17to24(__m256i input) -> __m256i { using tables = pack_tables_avx2_24; constexpr u32 bitmask = (1 << BIT_WIDTH) - 1; const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(bitmask)); @@ -366,7 +366,7 @@ __always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m25 */ template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { +auto mm_pack_aligned_epi32_avx2(__m128i input) -> __m128i { if constexpr (BIT_WIDTH == 8) { return _mm_packus_epi16(_mm_packs_epi32(input, _mm_setzero_si128()), _mm_setzero_si128()); } else { @@ -379,7 +379,7 @@ auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 16) -auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { +auto mm_pack_epi32_avx2(__m128i input) -> __m128i { if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { return internal::mm_pack_epi32_avx2_1to3(input); } else if constexpr (BIT_WIDTH >= 4 && BIT_WIDTH <= 8) { @@ -397,7 +397,7 @@ auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { */ template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -__m256i mm256_pack_aligned_epi32_avx2(const __m256i& input) { +__m256i mm256_pack_aligned_epi32_avx2(__m256i input) { if constexpr (BIT_WIDTH == 8) { const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); @@ -412,7 +412,7 @@ __m256i mm256_pack_aligned_epi32_avx2(const __m256i& input) { */ template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__m256i mm256_pack_epi32_avx2(const __m256i& input) { +__m256i mm256_pack_epi32_avx2(__m256i input) { if constexpr (BIT_WIDTH == 8 || BIT_WIDTH == 16) { return internal::mm256_pack_aligned_epi32_avx2(input); } else { @@ -471,7 +471,7 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scal } if constexpr (remaining) { - constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; const __m256 source = _mm256_maskload_ps(input, internal::mm256_tail_mask_epi32()); const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); @@ -524,7 +524,7 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scal } if constexpr (remaining) { - constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; constexpr u8 first_lanes = remaining < 4 ? remaining : 4; constexpr u8 second_lanes = remaining > 4 ? remaining - 4 : 0; @@ -536,10 +536,10 @@ int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scal return _mm256_setzero_pd(); } }(); - const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); - const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); - __m256i combined = _mm256_castsi128_si256(quantized1); - combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); const __m256i packed_input = internal::mm256_clamp_signed_epi32(combined); const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); std::memcpy(output, &packed, tail_bytes); diff --git a/include/pernix/x86/avx2/avx2_decompression.h b/include/pernix/x86/avx2/avx2_decompression.h index c17fc6a..027b71f 100644 --- a/include/pernix/x86/avx2/avx2_decompression.h +++ b/include/pernix/x86/avx2/avx2_decompression.h @@ -30,7 +30,7 @@ __always_inline __m256i mm256_convert_vmask_epi64(const __mmask8 mask8) { /** * @brief Dequantize four 32-bit integers to floats. */ -__always_inline __m128 mm_dequantize_epi32(const __m128i& input, const __m128& scale) { +__always_inline __m128 mm_dequantize_epi32(__m128i input, __m128 scale) { const __m128 converted = _mm_cvtepi32_ps(input); return _mm_mul_ps(converted, scale); } @@ -48,7 +48,7 @@ __always_inline __m128d convert_epi64_pd(const __m128i v) { /** * @brief Dequantize two 64-bit integers to doubles. */ -__always_inline __m128d mm_dequantize_epi64_pd(const __m128i& input, const __m128d& scale) { +__always_inline __m128d mm_dequantize_epi64_pd(__m128i input, __m128d scale) { const __m128d converted = convert_epi64_pd(input); return _mm_mul_pd(converted, scale); } @@ -56,7 +56,7 @@ __always_inline __m128d mm_dequantize_epi64_pd(const __m128i& input, const __m12 /** * @brief Dequantize eight 32-bit integers to floats. */ -__always_inline __m256 mm256_dequantize_epi32(const __m256i& input, const __m256& scale) { +__always_inline __m256 mm256_dequantize_epi32(__m256i input, __m256 scale) { const __m256 converted = _mm256_cvtepi32_ps(input); return _mm256_mul_ps(converted, scale); } @@ -74,7 +74,7 @@ __always_inline __m256d convert_epi64_pd(__m256i v) { /** * @brief Dequantize four 64-bit integers to doubles. */ -__always_inline __m256d mm256_dequantize_epi64_pd(const __m256i& input, const __m256d& scale) { +__always_inline __m256d mm256_dequantize_epi64_pd(__m256i input, __m256d scale) { const __m256d converted = convert_epi64_pd(input); return _mm256_mul_pd(converted, scale); } diff --git a/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h index 39ef6c0..1f35579 100644 --- a/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h +++ b/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h @@ -41,17 +41,17 @@ static __always_inline __m512i mm512_clamp_signed_epi64(__m512i input) { * @brief Quantize sixteen float values to 32-bit integers. */ -static __always_inline __m512i mm512_quantize_ps_epi32(const __m512& input, const __m512& scale) { +static __always_inline __m512i mm512_quantize_ps_epi32(__m512 input, __m512 scale) { const __m512 scaled = _mm512_mul_ps(input, scale); return _mm512_cvtps_epi32(scaled); } -static __always_inline __m512i mm512_quantize_pd_epi64(const __m512d& input, const __m512d& scale) { +static __always_inline __m512i mm512_quantize_pd_epi64(__m512d input, __m512d scale) { const __m512d scaled = _mm512_mul_pd(input, scale); return _mm512_cvtpd_epi64(scaled); } -static __always_inline __m256i mm512_quantize_pd_epi32(const __m512d& input, const __m512d& scale) { +static __always_inline __m256i mm512_quantize_pd_epi32(__m512d input, __m512d scale) { const __m512d scaled = _mm512_mul_pd(input, scale); return _mm512_cvtpd_epi32(scaled); } diff --git a/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h index 22a1fb7..49eb214 100644 --- a/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h +++ b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h @@ -16,12 +16,12 @@ namespace internal { /** * @brief Dequantize sixteen integer values to floats. */ -__always_inline __m512 mm512_dequantize_epi32(const __m512i& input, const __m512& scale) { +__always_inline __m512 mm512_dequantize_epi32(__m512i input, __m512 scale) { const __m512 converted = _mm512_cvtepi32_ps(input); return _mm512_mul_ps(converted, scale); } -__always_inline __m512d mm512_dequantize_epi64(const __m512i& input, const __m512d& scale) { +__always_inline __m512d mm512_dequantize_epi64(__m512i input, __m512d scale) { const __m512d converted = _mm512_cvtepi64_pd(input); return _mm512_mul_pd(converted, scale); } diff --git a/include/pernix/x86/avx512vbmi/unpacking.h b/include/pernix/x86/avx512vbmi/unpacking.h index 1426460..08c5d0a 100644 --- a/include/pernix/x86/avx512vbmi/unpacking.h +++ b/include/pernix/x86/avx512vbmi/unpacking.h @@ -53,7 +53,7 @@ __always_inline static __m128i _mm_srai_epi8(const __m128i a, const i8 imm8) { template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i& input) { +__always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(__m128i input) { if constexpr (BIT_WIDTH == 8) { return input; } else { @@ -85,7 +85,7 @@ __always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i& input) { template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i& input) { +__always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(__m128i input) { if constexpr (BIT_WIDTH == 16) { return input; } else { @@ -116,7 +116,7 @@ __always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i& input) { template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i& input) { +__always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(__m128i input) { using tables = detail::unpack_table; const __m128i permuted = _mm_permutexvar_epi8(load_table<__m128i>(tables::primary_permute), input); @@ -182,7 +182,7 @@ __always_inline static __m256i _mm256_srai_epi8(const __m256i a, const i8 imm8) template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i& input) { +__always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(__m256i input) { if constexpr (BIT_WIDTH == 8) { return input; } else { @@ -214,7 +214,7 @@ __always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i& input) template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i& input) { +__always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(__m256i input) { if constexpr (BIT_WIDTH == 16) { return input; } else { @@ -245,7 +245,7 @@ __always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i& input template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i& input) { +__always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(__m256i input) { using tables = detail::unpack_table; const __m256i permuted = _mm256_permutexvar_epi8(load_table<__m256i>(tables::primary_permute), input); @@ -311,7 +311,7 @@ __always_inline static __m512i _mm512_srai_epi8(const __m512i a, const i8 imm8) template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i& input) { +__always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(__m512i input) { if constexpr (BIT_WIDTH == 8) { return input; } else { @@ -343,7 +343,7 @@ __always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i& input) template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i& input) { +__always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(__m512i input) { if constexpr (BIT_WIDTH == 16) { return input; } else { @@ -373,7 +373,7 @@ __always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i& input template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i& input) { +__always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(__m512i input) { using tables = detail::unpack_table; const __m512i permuted = _mm512_permutexvar_epi8(load_table<__m512i>(tables::primary_permute), input); diff --git a/include/pernix/x86/bmi2/bmi2_compression.h b/include/pernix/x86/bmi2/bmi2_compression.h index 57e73b7..f4f4584 100644 --- a/include/pernix/x86/bmi2/bmi2_compression.h +++ b/include/pernix/x86/bmi2/bmi2_compression.h @@ -50,7 +50,7 @@ static constexpr std::tuple pack_avx2_bmi2_constants() { */ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) -static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { +static inline auto mm_pack_epi32_bmi2(__m128i input) -> __m128i { const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 16) { @@ -81,7 +81,7 @@ static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { */ template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { +static inline auto mm256_pack_epi32_bmi2(__m256i input) -> __m256i { const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { @@ -186,7 +186,7 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scal } if constexpr (remaining) { - constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; const __m256 source = _mm256_maskload_ps(input, internal::mm256_tail_mask_epi32()); const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); @@ -236,7 +236,7 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scal } if constexpr (remaining) { - constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; constexpr u8 first_lanes = remaining < 4 ? remaining : 4; constexpr u8 second_lanes = remaining > 4 ? remaining - 4 : 0; @@ -252,7 +252,7 @@ int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scal const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); __m256i combined = _mm256_castsi128_si256(quantized1); combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_bmi2(internal::mm256_clamp_signed_epi32(combined)); + const __m256i packed = internal::mm256_pack_epi32_bmi2(internal::mm256_clamp_signed_epi32(combined)); std::memcpy(output, &packed, tail_bytes); } return 0; From 4b81622218614eb28a376b7b46966d34b32d3635 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 16 Jul 2026 20:37:11 +0200 Subject: [PATCH 40/43] feat: update clang format and tidy configurations for improved code style and diagnostics --- .clang-format | 126 ++++++++++++++++++++++++++++++++++++++++++++++++-- .clang-tidy | 109 +++++++++++++++++++++++++++++++++++++++---- .clangd | 69 +++++++++++++++++++++++++++ 3 files changed, 289 insertions(+), 15 deletions(-) diff --git a/.clang-format b/.clang-format index 3c83445..17587f3 100644 --- a/.clang-format +++ b/.clang-format @@ -1,11 +1,127 @@ +--- Language: Cpp BasedOnStyle: Google -AccessModifierOffset: -4 -AlignConsecutiveAssignments: true -AllowShortFunctionsOnASingleLine: InlineOnly +# General layout ColumnLimit: 140 +IndentWidth: 4 +ContinuationIndentWidth: 4 +TabWidth: 4 +UseTab: Never + +# Braces and compact constructs +BreakBeforeBraces: Attach +AllowShortBlocksOnASingleLine: Empty +AllowShortCaseExpressionOnASingleLine: true +AllowShortEnumsOnASingleLine: false +AllowShortFunctionsOnASingleLine: InlineOnly +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AllowShortLambdasOnASingleLine: Inline + +# Alignment +AlignAfterOpenBracket: true +AlignArrayOfStructures: Left +AlignConsecutiveAssignments: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false +AlignConsecutiveDeclarations: + Enabled: false +AlignConsecutiveMacros: + Enabled: false +AlignOperands: Align +AlignTrailingComments: + Kind: Always + OverEmptyLines: 0 + +# Pointers and references DerivePointerAlignment: false +PointerAlignment: Left +ReferenceAlignment: Pointer + +# Function declarations and calls +BinPackArguments: true +BinPackParameters: BinPack +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +PenaltyBreakBeforeFirstCallParameter: 19 + +# Templates +AlwaysBreakTemplateDeclarations: MultiLine +BreakTemplateDeclarations: MultiLine + +# Constructor initializers +BreakConstructorInitializers: BeforeColon +ConstructorInitializerIndentWidth: 4 +PackConstructorInitializers: BinPack + +# Operators and expressions +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakStringLiterals: true +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpacesInAngles: Never +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false + +# Includes +SortIncludes: CaseSensitive +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^' + Priority: 2 + SortPriority: 2 + + - Regex: '^<' + Priority: 3 + SortPriority: 3 + + - Regex: '^"' + Priority: 4 + SortPriority: 4 + + +# Namespaces +NamespaceIndentation: None FixNamespaceComments: true -IndentWidth: 4 -PointerAlignment: Left \ No newline at end of file +CompactNamespaces: false +ShortNamespaceLines: 1 + +# Access modifiers +AccessModifierOffset: -4 +EmptyLineBeforeAccessModifier: LogicalBlock +EmptyLineAfterAccessModifier: Never + +# Preprocessor +IndentPPDirectives: None +PPIndentWidth: 4 +IndentCaseLabels: true +IndentCaseBlocks: false + +# Comments +ReflowComments: Always +SpacesBeforeTrailingComments: 2 + +# Miscellaneous +InsertBraces: false +InsertNewlineAtEOF: true +KeepEmptyLines: + AtEndOfFile: false + AtStartOfBlock: false + AtStartOfFile: false +MaxEmptyLinesToKeep: 1 +SeparateDefinitionBlocks: Always +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope \ No newline at end of file diff --git a/.clang-tidy b/.clang-tidy index fe218fe..20ea7a3 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,13 +1,102 @@ +--- Checks: > - bugprone-* - cppcoreguidelines-* - clang-analyzer-* - modernize-* + -*, + clang-analyzer-*, + bugprone-*, + performance-*, + portability-*, + misc-*, + modernize-*, + readability-*, + cppcoreguidelines-*, + + -bugprone-easily-swappable-parameters, + -bugprone-narrowing-conversions, + -bugprone-signed-char-misuse, + + -cppcoreguidelines-avoid-c-arrays, + -cppcoreguidelines-avoid-do-while, + -cppcoreguidelines-avoid-magic-numbers, + -cppcoreguidelines-avoid-non-const-global-variables, + -cppcoreguidelines-init-variables, + -cppcoreguidelines-macro-usage, + -cppcoreguidelines-non-private-member-variables-in-classes-owning-memory, + -cppcoreguidelines-pro-bounds-constant-array-index, + -cppcoreguidelines-pro-bounds-pointer-arithmetic, + -cppcoreguidelines-pro-type-cstyle-cast, + -cppcoreguidelines-pro-type-reinterpret-cast, + -cppcoreguidelines-pro-type-union-access, + + -misc-include-cleaner, + -misc-no-recursion, + + -modernize-avoid-c-arrays, + -modernize-concat-nested-namespaces, + -modernize-loop-convert, + -modernize-return-braced-init-list, + -modernize-use-auto, + -modernize-use-nodiscard, + -modernize-use-trailing-return-type, + + -performance-avoid-endl, + + -readability-avoid-const-params-in-decls, + -readability-braces-around-statements, + -readability-convert-member-functions-to-static, + -readability-else-after-return, + -readability-function-cognitive-complexity, + -readability-function-size, + -readability-identifier-length, + -readability-identifier-naming, + -readability-implicit-bool-conversion, + -readability-isolate-declaration, + -readability-magic-numbers, + -readability-make-member-function-const, + -readability-named-parameter, + -readability-redundant-access-specifiers, + -readability-static-accessed-through-instance + +WarningsAsErrors: > + clang-analyzer-*, + bugprone-*, performance-* - readability-* - misc-* - -modernize-use-trailing-return-type -WarningsAsErrors: 'bugprone-*,clang-analyzer-*' -HeaderFilterRegex: '' -FormatStyle: none \ No newline at end of file +HeaderFilterRegex: '^(include|src|tests|benchmarks)/' +ExcludeHeaderFilterRegex: > + (^|/)(build|cmake-build-[^/]*|third_party|external|vendor|generated)/ + +SystemHeaders: false +FormatStyle: file +UseColor: true + +CheckOptions: + # Suspiciously adjacent parameters are common in low-level SIMD helpers. + bugprone-easily-swappable-parameters.MinimumLength: 3 + bugprone-easily-swappable-parameters.IgnoredParameterNames: '"",begin,end,first,last,lhs,rhs,x,y,z' + + # Treat assertions as side-effecting expressions. + bugprone-assert-side-effect.AssertMacros: 'assert,PERNIX_ASSERT' + + # Permit common test and benchmark patterns. + bugprone-unchecked-optional-access.CheckOsLinuxAndMac: true + + # Do not rewrite auto when it improves readability or hides intrinsic types. + modernize-use-auto.MinTypeNameLength: 12 + modernize-use-auto.RemoveStars: false + + # Allow compact SIMD and mathematical variable names. + readability-identifier-length.MinimumVariableNameLength: 1 + readability-identifier-length.MinimumParameterNameLength: 1 + readability-identifier-length.IgnoredVariableNames: '^[ijknxyz]|lo|hi|src|dst|tmp|mask|count|scale$' + readability-identifier-length.IgnoredParameterNames: '^[ijknxyz]|lo|hi|src|dst|input|output|count|scale$' + + # Keep function complexity checks informative if enabled manually. + readability-function-cognitive-complexity.Threshold: 40 + readability-function-cognitive-complexity.DescribeBasicIncrements: true + + # Ignore conventional bit-oriented numeric literals. + readability-magic-numbers.IgnoredIntegerValues: > + 0;1;2;3;4;7;8;15;16;24;31;32;63;64;127;128;255;256;511;512 + readability-magic-numbers.IgnorePowersOf2IntegerValues: true + readability-magic-numbers.IgnoreTypeAliases: true + readability-magic-numbers.IgnoreBitFieldsWidths: true \ No newline at end of file diff --git a/.clangd b/.clangd index e69de29..368ac08 100644 --- a/.clangd +++ b/.clangd @@ -0,0 +1,69 @@ +--- +CompileFlags: + CompilationDatabase: build + Add: + - -Wall + - -Wextra + - -Wpedantic + - -Wconversion + - -Wsign-conversion + - -Wshadow + - -Wundef + - -Wdouble-promotion + - -Wformat=2 + - -Wimplicit-fallthrough + - -Wmissing-declarations + - -Wmissing-prototypes + - -Wnon-virtual-dtor + - -Wold-style-cast + - -Woverloaded-virtual + - -Wzero-as-null-pointer-constant + Remove: + - -Werror + - -Wno-* + +Diagnostics: + Suppress: + - pp_including_mainfile_in_preamble + + ClangTidy: + Add: + - clang-analyzer-* + - bugprone-* + - performance-* + - portability-* + + Remove: + - bugprone-easily-swappable-parameters + - bugprone-narrowing-conversions + - bugprone-signed-char-misuse + + FastCheckFilter: Strict + + UnusedIncludes: Strict + MissingIncludes: Strict + +Index: + Background: Build + StandardLibrary: true + +Completion: + AllScopes: true + ArgumentLists: FullPlaceholders + HeaderInsertion: IWYU + CodePatterns: All + +Hover: + ShowAKA: true + +InlayHints: + Enabled: true + ParameterNames: true + DeducedTypes: true + Designators: true + BlockEnd: false + TypeNameLimit: 32 + +SemanticTokens: + DisabledKinds: [] + DisabledModifiers: [] \ No newline at end of file From 10fd0927ff636685d6b243209afed0b421276025 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 16 Jul 2026 20:41:38 +0200 Subject: [PATCH 41/43] refactor: reorganize include directives for improved clarity and consistency --- .clang-format | 2 +- include/pernix/arm64/neon/common.h | 94 +++++---- include/pernix/backend.hpp | 4 +- include/pernix/detail/api.hpp | 2 +- include/pernix/dispatch/cpu_features.h | 4 +- include/pernix/dispatch/select.h | 3 +- include/pernix/dispatch/select_impl.h | 196 ++++++++++-------- include/pernix/pernix.hpp | 3 +- include/pernix/x86/avx2/avx2_tables.h | 6 + include/pernix/x86/avx512vbmi/packing.h | 12 +- src/arm64/neon/decompression.cpp | 28 +-- src/arm64/sve2/decompression.cpp | 28 +-- src/fallback/fallback_decompression.cpp | 28 +-- src/fallback/stdpar_decompression.cpp | 40 ++-- src/x86/avx2/avx2_decompression.cpp | 28 +-- .../avx512vbmi/avx512vbmi_decompression.cpp | 28 +-- src/x86/bmi2/bmi2_decompression.cpp | 28 +-- tests/c_api_roundtrip.c | 6 +- tests/c_api_validation.c | 9 +- tests/deterministic_correctness_tests.cpp | 52 +++-- tests/fallback_tests.cpp | 19 +- tests/header_only_tests.cpp | 18 +- tests/include/testset.h | 3 +- tests/install_consumer/main.c | 7 +- tests/install_consumer/main.cpp | 3 +- tests/simd_tests.cpp | 3 +- 26 files changed, 379 insertions(+), 275 deletions(-) diff --git a/.clang-format b/.clang-format index 17587f3..69ed367 100644 --- a/.clang-format +++ b/.clang-format @@ -53,7 +53,7 @@ PenaltyBreakBeforeFirstCallParameter: 19 # Templates AlwaysBreakTemplateDeclarations: MultiLine -BreakTemplateDeclarations: MultiLine +BreakTemplateDeclarations: Yes # Constructor initializers BreakConstructorInitializers: BeforeColon diff --git a/include/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h index be0d23e..63d15bb 100644 --- a/include/pernix/arm64/neon/common.h +++ b/include/pernix/arm64/neon/common.h @@ -20,35 +20,43 @@ __always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(int8x16_t input) { const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); - return {{ - vmovl_s16(vget_low_s16(s16_lo)), - vmovl_s16(vget_high_s16(s16_lo)), - vmovl_s16(vget_low_s16(s16_hi)), - vmovl_s16(vget_high_s16(s16_hi)), - }}; + return { + { + vmovl_s16(vget_low_s16(s16_lo)), + vmovl_s16(vget_high_s16(s16_lo)), + vmovl_s16(vget_low_s16(s16_hi)), + vmovl_s16(vget_high_s16(s16_hi)), + } + }; } __always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(int16x8_t input) { - return {{ - vmovl_s16(vget_low_s16(input)), - vmovl_s16(vget_high_s16(input)), - }}; + return { + { + vmovl_s16(vget_low_s16(input)), + vmovl_s16(vget_high_s16(input)), + } + }; } __always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, float32x4_t scale) { - return {{ - vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), - }}; + return { + { + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), + } + }; } __always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t& input, float32x4_t scale) { - return {{ - vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), - vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), - }}; + return { + { + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + } + }; } __always_inline float32x4_t neon_dequantize_epi32(int32x4_t input, float32x4_t scale) { @@ -60,22 +68,26 @@ __always_inline float64x2_t neon_dequantize_epi32_f64(int32x2_t input, float64x2 } __always_inline float64x2x2_t neon_dequantize_epi32_f64(int32x4_t input, float64x2_t scale) { - return {{ - neon_dequantize_epi32_f64(vget_low_s32(input), scale), - neon_dequantize_epi32_f64(vget_high_s32(input), scale), - }}; + return { + { + neon_dequantize_epi32_f64(vget_low_s32(input), scale), + neon_dequantize_epi32_f64(vget_high_s32(input), scale), + } + }; } __always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t& input, float64x2_t scale) { const float64x2x2_t dequantized_low = neon_dequantize_epi32_f64(input.val[0], scale); const float64x2x2_t dequantized_high = neon_dequantize_epi32_f64(input.val[1], scale); - return {{ - dequantized_low.val[0], - dequantized_low.val[1], - dequantized_high.val[0], - dequantized_high.val[1], - }}; + return { + { + dequantized_low.val[0], + dequantized_low.val[1], + dequantized_high.val[0], + dequantized_high.val[1], + } + }; } __always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input, float64x2_t scale) { @@ -84,16 +96,18 @@ __always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input const float64x2x2_t dequantized2 = neon_dequantize_epi32_f64(input.val[2], scale); const float64x2x2_t dequantized3 = neon_dequantize_epi32_f64(input.val[3], scale); - return {{ - dequantized0.val[0], - dequantized0.val[1], - dequantized1.val[0], - dequantized1.val[1], - dequantized2.val[0], - dequantized2.val[1], - dequantized3.val[0], - dequantized3.val[1], - }}; + return { + { + dequantized0.val[0], + dequantized0.val[1], + dequantized1.val[0], + dequantized1.val[1], + dequantized2.val[0], + dequantized2.val[1], + dequantized3.val[0], + dequantized3.val[1], + } + }; } __always_inline uint8x16_t neon_load_tail_elements_int8(const u8* input, const u32 tail_bytes_count) { diff --git a/include/pernix/backend.hpp b/include/pernix/backend.hpp index 7e9d4bc..228b1d1 100644 --- a/include/pernix/backend.hpp +++ b/include/pernix/backend.hpp @@ -14,8 +14,8 @@ enum class Backend { Arm64Neon = PERNIX_BACKEND_ARM64_NEON, Arm64Sve = PERNIX_BACKEND_ARM64_SVE, FallbackStdpar = PERNIX_BACKEND_FALLBACK_STDPAR, - FallbackSimd = PERNIX_BACKEND_FALLBACK_SIMD + FallbackSimd = PERNIX_BACKEND_FALLBACK_SIMD, }; -} +} // namespace pernix #endif // PERNIX_BACKEND_HPP diff --git a/include/pernix/detail/api.hpp b/include/pernix/detail/api.hpp index 04a11a4..41046c6 100644 --- a/include/pernix/detail/api.hpp +++ b/include/pernix/detail/api.hpp @@ -1,11 +1,11 @@ #ifndef PERNIX_DETAIL_API_HPP #define PERNIX_DETAIL_API_HPP +#include #include #include #include -#include #include #include diff --git a/include/pernix/dispatch/cpu_features.h b/include/pernix/dispatch/cpu_features.h index fb797e4..4d6e866 100644 --- a/include/pernix/dispatch/cpu_features.h +++ b/include/pernix/dispatch/cpu_features.h @@ -48,7 +48,9 @@ inline CpuFeatures detect_cpu_features() { #if defined(_MSC_VER) __cpuidex(regs, 1, 0); - const auto xgetbv = [](const unsigned int index) -> u64 { return _xgetbv(index); }; + const auto xgetbv = [](const unsigned int index) -> u64 { + return _xgetbv(index); + }; #else __cpuid_count(1, 0, regs[0], regs[1], regs[2], regs[3]); const auto xgetbv = [](const unsigned int index) -> u64 { diff --git a/include/pernix/dispatch/select.h b/include/pernix/dispatch/select.h index 8e723a2..b3c8334 100644 --- a/include/pernix/dispatch/select.h +++ b/include/pernix/dispatch/select.h @@ -1,9 +1,8 @@ #ifndef PERNIX_SELECT_H #define PERNIX_SELECT_H -#include - #include +#include #if !defined(PERNIX_BUILD_X86_AVX2) && !defined(PERNIX_DISABLE_AVX2) #if defined(PERNIX_USE_SIMDE) || defined(__AVX2__) diff --git a/include/pernix/dispatch/select_impl.h b/include/pernix/dispatch/select_impl.h index 0ca59f1..8737320 100644 --- a/include/pernix/dispatch/select_impl.h +++ b/include/pernix/dispatch/select_impl.h @@ -1351,24 +1351,28 @@ inline Kernel select_fallback_simd_compress_blocks_f64(cons } // namespace pernix::internal namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_block_fallback); \ return Kernel("fallback", &decompress_block_fallback) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_blocks_fallback); \ return Kernel("fallback", &decompress_blocks_fallback) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_block_fallback); \ return Kernel("fallback", &decompress_block_fallback) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_blocks_fallback); \ return Kernel("fallback", &decompress_blocks_fallback) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ case BS: \ @@ -1566,24 +1570,28 @@ inline Kernel select_fallback_scalar_decompress_blocks_f64( return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); } -#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) -#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) -#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) -#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(BS) \ @@ -2049,24 +2057,28 @@ inline Kernel select_avx2_compress_blocks_f64(const u8 bit_ } // namespace pernix::internal namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_block_avx2); \ return Kernel("avx2", &mm256_decompress_block_avx2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_blocks_avx2); \ return Kernel("avx2", &mm256_decompress_blocks_avx2) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_block_avx2); \ return Kernel("avx2", &mm256_decompress_block_avx2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_blocks_avx2); \ return Kernel("avx2", &mm256_decompress_blocks_avx2) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ case BS: \ @@ -2464,24 +2476,28 @@ inline Kernel select_bmi2_compress_blocks_f64(const u8 bit_ } // namespace pernix::internal namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_block_bmi2); \ return Kernel("bmi2", &mm256_decompress_block_bmi2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ return Kernel("bmi2", &mm256_decompress_blocks_bmi2) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_block_bmi2); \ return Kernel("bmi2", &mm256_decompress_block_bmi2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ return Kernel("bmi2", &mm256_decompress_blocks_bmi2) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ case BS: \ @@ -2871,24 +2887,28 @@ inline Kernel select_avx512vbmi_compress_blocks_f64(const u } // namespace pernix::internal namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ case BS: \ @@ -3108,24 +3128,28 @@ using pernix::arm64::neon::neon_decompress_block; using pernix::arm64::neon::neon_decompress_blocks; namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_block); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_block); \ return Kernel("neon", &neon_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_blocks); \ return Kernel("neon", &neon_decompress_blocks) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_block); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_block); \ return Kernel("neon", &neon_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_blocks); \ return Kernel("neon", &neon_decompress_blocks) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ @@ -3346,24 +3370,28 @@ using pernix::arm64::sve2::sve2_decompress_block; using pernix::arm64::sve2::sve2_decompress_blocks; namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_block); \ return Kernel("sve2", &sve2_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_blocks); \ return Kernel("sve2", &sve2_decompress_blocks) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_block); \ return Kernel("sve2", &sve2_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_blocks); \ return Kernel("sve2", &sve2_decompress_blocks) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ diff --git a/include/pernix/pernix.hpp b/include/pernix/pernix.hpp index 602e13a..2358659 100644 --- a/include/pernix/pernix.hpp +++ b/include/pernix/pernix.hpp @@ -1,9 +1,10 @@ #ifndef PERNIX_HPP #define PERNIX_HPP -#include #include #include + +#include #include namespace pernix { diff --git a/include/pernix/x86/avx2/avx2_tables.h b/include/pernix/x86/avx2/avx2_tables.h index 2ee2643..e8e5368 100644 --- a/include/pernix/x86/avx2/avx2_tables.h +++ b/include/pernix/x86/avx2/avx2_tables.h @@ -360,6 +360,7 @@ struct pack_tables_avx2_16 { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" + __always_inline static T get_permute1() { if constexpr (std::is_same_v) { return _mm256_load_si256(reinterpret_cast(permute1.data())); @@ -413,6 +414,7 @@ struct pack_tables_avx2_16 { } return T{}; } + #pragma GCC diagnostic pop }; @@ -647,6 +649,7 @@ struct pack_tables_avx2_24 { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" + __always_inline static T get_permute1() { if constexpr (std::is_same_v) { return _mm256_load_si256(reinterpret_cast(permute1.data())); @@ -700,6 +703,7 @@ struct pack_tables_avx2_24 { } return T{}; } + #pragma GCC diagnostic pop }; @@ -760,6 +764,7 @@ struct unpack_tables_avx2 { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" + __always_inline static __m256i get_permute() { return _mm256_load_si256(reinterpret_cast(permute.data())); } __always_inline static T get_shuffle() { @@ -777,6 +782,7 @@ struct unpack_tables_avx2 { return _mm256_load_si256(reinterpret_cast(shift.data())); } } + #pragma GCC diagnostic pop }; } // namespace pernix::internal diff --git a/include/pernix/x86/avx512vbmi/packing.h b/include/pernix/x86/avx512vbmi/packing.h index 5424458..c07946c 100644 --- a/include/pernix/x86/avx512vbmi/packing.h +++ b/include/pernix/x86/avx512vbmi/packing.h @@ -61,7 +61,9 @@ __always_inline __m128i mm_pack_table_avx512vbmi(const __m128i values) { } }; - const auto combine = [](__m128i left, __m128i right) { return _mm_or_si128(left, right); }; + const auto combine = [](__m128i left, __m128i right) { + return _mm_or_si128(left, right); + }; __m128i result = combine_primary_contributors(pack_contributor, combine, std::make_index_sequence{}); @@ -163,7 +165,9 @@ __always_inline __m256i mm256_pack_table_avx512vbmi(const __m256i values) { } }; - const auto combine = [](__m256i left, __m256i right) { return _mm256_or_si256(left, right); }; + const auto combine = [](__m256i left, __m256i right) { + return _mm256_or_si256(left, right); + }; __m256i result = combine_primary_contributors(pack_contributor, combine, std::make_index_sequence{}); @@ -265,7 +269,9 @@ __always_inline __m512i mm512_pack_table_avx512vbmi(const __m512i values) { } }; - const auto combine = [](__m512i left, __m512i right) { return _mm512_or_si512(left, right); }; + const auto combine = [](__m512i left, __m512i right) { + return _mm512_or_si512(left, right); + }; __m512i result = combine_primary_contributors(pack_contributor, combine, std::make_index_sequence{}); diff --git a/src/arm64/neon/decompression.cpp b/src/arm64/neon/decompression.cpp index 736498c..3acb4bb 100644 --- a/src/arm64/neon/decompression.cpp +++ b/src/arm64/neon/decompression.cpp @@ -5,24 +5,28 @@ using pernix::arm64::neon::neon_decompress_block; using pernix::arm64::neon::neon_decompress_blocks; namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_block); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_block); \ return Kernel("neon", &neon_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_blocks); \ return Kernel("neon", &neon_decompress_blocks) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_block); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_block); \ return Kernel("neon", &neon_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("neon", &neon_decompress_blocks); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_blocks); \ return Kernel("neon", &neon_decompress_blocks) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ diff --git a/src/arm64/sve2/decompression.cpp b/src/arm64/sve2/decompression.cpp index 16c8e35..418a2ea 100644 --- a/src/arm64/sve2/decompression.cpp +++ b/src/arm64/sve2/decompression.cpp @@ -5,24 +5,28 @@ using pernix::arm64::sve2::sve2_decompress_block; using pernix::arm64::sve2::sve2_decompress_blocks; namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_block); \ return Kernel("sve2", &sve2_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_blocks); \ return Kernel("sve2", &sve2_decompress_blocks) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_block); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_block); \ return Kernel("sve2", &sve2_decompress_block) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("sve2", &sve2_decompress_blocks); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_blocks); \ return Kernel("sve2", &sve2_decompress_blocks) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ diff --git a/src/fallback/fallback_decompression.cpp b/src/fallback/fallback_decompression.cpp index 44e8db0..dfedb75 100644 --- a/src/fallback/fallback_decompression.cpp +++ b/src/fallback/fallback_decompression.cpp @@ -2,24 +2,28 @@ #include namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_block_fallback); \ return Kernel("fallback", &decompress_block_fallback) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_blocks_fallback); \ return Kernel("fallback", &decompress_blocks_fallback) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback", &decompress_block_fallback); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_block_fallback); \ return Kernel("fallback", &decompress_block_fallback) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback", &decompress_blocks_fallback); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_blocks_fallback); \ return Kernel("fallback", &decompress_blocks_fallback) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ diff --git a/src/fallback/stdpar_decompression.cpp b/src/fallback/stdpar_decompression.cpp index 9bd51ce..ec0dd7b 100644 --- a/src/fallback/stdpar_decompression.cpp +++ b/src/fallback/stdpar_decompression.cpp @@ -2,24 +2,28 @@ #include namespace pernix::internal { -#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) -#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) -#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) -#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) #define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(BS) \ @@ -146,8 +150,7 @@ namespace pernix::internal { return {"fallback_stdpar", nullptr}; \ } -Kernel select_fallback_stdpar_decompress_block_f32(const u8 bit_width, const u32 block_size, - const bool sign_values) { +Kernel select_fallback_stdpar_decompress_block_f32(const u8 bit_width, const u32 block_size, const bool sign_values) { switch (block_size) { PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(64); PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(128); @@ -159,8 +162,7 @@ Kernel select_fallback_stdpar_decompress_block_f32(const u8 } } -Kernel select_fallback_stdpar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, - const bool sign_values) { +Kernel select_fallback_stdpar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, const bool sign_values) { switch (block_size) { PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(64); PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(128); @@ -172,8 +174,7 @@ Kernel select_fallback_stdpar_decompress_blocks_f32(const u } } -Kernel select_fallback_stdpar_decompress_block_f64(const u8 bit_width, const u32 block_size, - const bool sign_values) { +Kernel select_fallback_stdpar_decompress_block_f64(const u8 bit_width, const u32 block_size, const bool sign_values) { switch (block_size) { PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(64); PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(128); @@ -185,8 +186,7 @@ Kernel select_fallback_stdpar_decompress_block_f64(const u8 } } -Kernel select_fallback_stdpar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, - const bool sign_values) { +Kernel select_fallback_stdpar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, const bool sign_values) { switch (block_size) { PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(64); PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(128); diff --git a/src/x86/avx2/avx2_decompression.cpp b/src/x86/avx2/avx2_decompression.cpp index 05b530e..4629ba1 100644 --- a/src/x86/avx2/avx2_decompression.cpp +++ b/src/x86/avx2/avx2_decompression.cpp @@ -2,24 +2,28 @@ #include namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_block_avx2); \ return Kernel("avx2", &mm256_decompress_block_avx2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_blocks_avx2); \ return Kernel("avx2", &mm256_decompress_blocks_avx2) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_block_avx2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_block_avx2); \ return Kernel("avx2", &mm256_decompress_block_avx2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx2", &mm256_decompress_blocks_avx2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_blocks_avx2); \ return Kernel("avx2", &mm256_decompress_blocks_avx2) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ case BS: \ diff --git a/src/x86/avx512vbmi/avx512vbmi_decompression.cpp b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp index 4ebb3b7..a3dbc13 100644 --- a/src/x86/avx512vbmi/avx512vbmi_decompression.cpp +++ b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp @@ -2,24 +2,28 @@ #include namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ case BS: \ diff --git a/src/x86/bmi2/bmi2_decompression.cpp b/src/x86/bmi2/bmi2_decompression.cpp index e3848e9..d02fc3b 100644 --- a/src/x86/bmi2/bmi2_decompression.cpp +++ b/src/x86/bmi2/bmi2_decompression.cpp @@ -2,24 +2,28 @@ #include namespace pernix::internal { -#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_block_bmi2); \ return Kernel("bmi2", &mm256_decompress_block_bmi2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ - case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ return Kernel("bmi2", &mm256_decompress_blocks_bmi2) -#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_block_bmi2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_block_bmi2); \ return Kernel("bmi2", &mm256_decompress_block_bmi2) -#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ - case N: \ - if (sign_values) return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ return Kernel("bmi2", &mm256_decompress_blocks_bmi2) #define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ case BS: \ diff --git a/tests/c_api_roundtrip.c b/tests/c_api_roundtrip.c index 433c31e..0ffe099 100644 --- a/tests/c_api_roundtrip.c +++ b/tests/c_api_roundtrip.c @@ -1,7 +1,11 @@ #include int main(void) { - enum { bit_width = 16, block_size = 64, elements = (block_size * 8) / bit_width }; + enum { + bit_width = 16, + block_size = 64, + elements = (block_size * 8) / bit_width + }; if (pernix_min_bit_width() != 1 || pernix_max_bit_width() != 24) { return 1; diff --git a/tests/c_api_validation.c b/tests/c_api_validation.c index 0e07be7..ad03608 100644 --- a/tests/c_api_validation.c +++ b/tests/c_api_validation.c @@ -1,8 +1,13 @@ -#include #include + +#include #include -enum { bit_width = 8, block_size = 64, elements = (block_size * 8) / bit_width }; +enum { + bit_width = 8, + block_size = 64, + elements = (block_size * 8) / bit_width +}; static int expect_status(pernix_status actual, pernix_status expected) { return actual == expected ? 0 : 1; diff --git a/tests/deterministic_correctness_tests.cpp b/tests/deterministic_correctness_tests.cpp index 0d43e56..9dfabaf 100644 --- a/tests/deterministic_correctness_tests.cpp +++ b/tests/deterministic_correctness_tests.cpp @@ -1,13 +1,13 @@ -#include +#include #include #include #include #include -#include #include #include #include +#include #include #include @@ -16,31 +16,41 @@ namespace { constexpr u32 kBlockSize = 64; constexpr u32 kBlocks = 3; -enum class Pattern { Zero, Constant, Increasing, Decreasing, SmallMagnitude, LargeMagnitude, QuantizationBoundary, PseudoRandom }; +enum class Pattern { + Zero, + Constant, + Increasing, + Decreasing, + SmallMagnitude, + LargeMagnitude, + QuantizationBoundary, + PseudoRandom +}; struct PatternCase { Pattern pattern; std::string_view name; }; -constexpr std::array kPatternCases{{ - {Pattern::Zero, "zero"}, - {Pattern::Constant, "constant"}, - {Pattern::Increasing, "increasing"}, - {Pattern::Decreasing, "decreasing"}, - {Pattern::SmallMagnitude, "small_magnitude"}, - {Pattern::LargeMagnitude, "large_magnitude"}, - {Pattern::QuantizationBoundary, "quantization_boundary"}, - {Pattern::PseudoRandom, "pseudo_random"}, -}}; - -constexpr std::array kComparisonBackends{{ - PERNIX_BACKEND_X86_AVX2, - PERNIX_BACKEND_X86_BMI2, - PERNIX_BACKEND_X86_AVX512_VBMI, - PERNIX_BACKEND_ARM64_NEON, - PERNIX_BACKEND_ARM64_SVE, -}}; +constexpr std::array kPatternCases{ + { + {Pattern::Zero, "zero"}, + {Pattern::Constant, "constant"}, + {Pattern::Increasing, "increasing"}, + {Pattern::Decreasing, "decreasing"}, + {Pattern::SmallMagnitude, "small_magnitude"}, + {Pattern::LargeMagnitude, "large_magnitude"}, + {Pattern::QuantizationBoundary, "quantization_boundary"}, + {Pattern::PseudoRandom, "pseudo_random"}, + } +}; + +constexpr std::array kComparisonBackends{ + { + PERNIX_BACKEND_X86_AVX2, PERNIX_BACKEND_X86_BMI2, + PERNIX_BACKEND_X86_AVX512_VBMI, PERNIX_BACKEND_ARM64_NEON, + PERNIX_BACKEND_ARM64_SVE, } +}; template FloatT abs_value(const FloatT value) { diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp index f7629af..5f5f445 100644 --- a/tests/fallback_tests.cpp +++ b/tests/fallback_tests.cpp @@ -1,10 +1,9 @@ -#include - #include #include #include #include #include +#include #include #if defined(__linux__) && defined(__aarch64__) @@ -56,10 +55,10 @@ template void expect_backend_compress_blocks_matches_scalar(FixtureT& fixture, CompressFn compress_fn, const pernix_backend backend) { using ScaleType = std::remove_cvref_t; - const u32 blocks = fixture.testSet.numberOfBlocks; - const u32 elements_per_block = fixture.testSet.elementsPerBlock; - const u32 total_elements = blocks * elements_per_block; - const usize compressed_bytes = static_cast(blocks) * FixtureT::BlockSize; + const u32 blocks = fixture.testSet.numberOfBlocks; + const u32 elements_per_block = fixture.testSet.elementsPerBlock; + const u32 total_elements = blocks * elements_per_block; + const usize compressed_bytes = static_cast(blocks) * FixtureT::BlockSize; std::vector input(total_elements); for (u32 block = 0; block < blocks; ++block) { @@ -71,8 +70,8 @@ void expect_backend_compress_blocks_matches_scalar(FixtureT& fixture, CompressFn std::vector scalar_output(compressed_bytes); std::vector stdpar_output(compressed_bytes); - auto status = compress_fn(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, input.data(), scale_inv, scalar_output.data(), - blocks); + auto status = compress_fn(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, input.data(), scale_inv, + scalar_output.data(), blocks); ASSERT_EQ(status, PERNIX_STATUS_OK); status = compress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, input.data(), scale_inv, stdpar_output.data(), blocks); @@ -761,8 +760,8 @@ TEST(FallbackStdparEdgeTest, RepeatedExecutionIsDeterministic) { true); ASSERT_EQ(st, PERNIX_STATUS_OK); - st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, second_compressed.data(), 0.25f, - second_restored.data(), blocks, true); + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, second_compressed.data(), 0.25f, second_restored.data(), + blocks, true); ASSERT_EQ(st, PERNIX_STATUS_OK); EXPECT_EQ(second_compressed, first_compressed); diff --git a/tests/header_only_tests.cpp b/tests/header_only_tests.cpp index ea0ac12..a6e41e8 100644 --- a/tests/header_only_tests.cpp +++ b/tests/header_only_tests.cpp @@ -1,11 +1,12 @@ -#include +#include #include #include #include -#include #include +#include + namespace { constexpr u8 kBitWidth = 8; constexpr u32 kBlockSize = PERNIX_TEST_BLOCK_SIZE; @@ -140,8 +141,8 @@ void expect_single_block_partial_group_matches_scalar(const u8 bit_width) { std::vector scalar_restored(elements_per_block, static_cast(0)); std::vector stdpar_restored(elements_per_block, static_cast(0)); - ASSERT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), static_cast(4), - std::span(scalar_compressed)), + ASSERT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), + static_cast(4), std::span(scalar_compressed)), PERNIX_STATUS_OK); ASSERT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), static_cast(4), std::span(stdpar_compressed)), @@ -329,10 +330,9 @@ TEST(HeaderOnlyPernix, FallbackStdparMatchesScalarForEdgeValues) { ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(scalar_compressed), 1.0f, std::span(scalar_signed), blocks, true), PERNIX_STATUS_OK); - ASSERT_EQ( - pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(stdpar_compressed), 1.0f, - std::span(stdpar_signed), blocks, true), - PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(stdpar_compressed), + 1.0f, std::span(stdpar_signed), blocks, true), + PERNIX_STATUS_OK); EXPECT_EQ(stdpar_signed, scalar_signed); } @@ -353,7 +353,7 @@ TEST(HeaderOnlyPernix, FallbackStdparCrossCompatibilityMatchesScalarForUnsignedD } TEST(HeaderOnlyPernix, FallbackStdparSignValuesFalseMatchesScalar) { - constexpr u8 bit_width = 4; + constexpr u8 bit_width = 4; const u32 elements_per_block = pernix::elements_per_block(bit_width); std::vector compressed(kBlockSize, 0); diff --git a/tests/include/testset.h b/tests/include/testset.h index 392ed44..87054af 100644 --- a/tests/include/testset.h +++ b/tests/include/testset.h @@ -1,7 +1,6 @@ #ifndef PERNIX_TESTSET_H #define PERNIX_TESTSET_H -#include #include #include @@ -14,6 +13,8 @@ #include #include +#include + #ifndef PERNIX_TEST_BLOCK_SIZE #define PERNIX_TEST_BLOCK_SIZE 64 #endif diff --git a/tests/install_consumer/main.c b/tests/install_consumer/main.c index 46e739f..b6a521a 100644 --- a/tests/install_consumer/main.c +++ b/tests/install_consumer/main.c @@ -1,7 +1,12 @@ #include int main(void) { - enum { bit_width = 8, block_size = 64, elements = (block_size * 8) / bit_width }; + enum { + bit_width = 8, + block_size = 64, + elements = (block_size * 8) / bit_width + }; + float input[elements]; float restored[elements]; u8 compressed[block_size]; diff --git a/tests/install_consumer/main.cpp b/tests/install_consumer/main.cpp index edf9ec0..0094d42 100644 --- a/tests/install_consumer/main.cpp +++ b/tests/install_consumer/main.cpp @@ -1,7 +1,8 @@ +#include + #include #include #include -#include int main() { constexpr u8 bit_width = 8; diff --git a/tests/simd_tests.cpp b/tests/simd_tests.cpp index 4ba86e3..0e01de5 100644 --- a/tests/simd_tests.cpp +++ b/tests/simd_tests.cpp @@ -1,6 +1,5 @@ -#include - #include +#include #include // --------------------------------------------------------------------------- From a126237729747bbf05209dd93ab69f23ae06d7a4 Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 16 Jul 2026 20:42:19 +0200 Subject: [PATCH 42/43] feat: add low_bit_mask function and validation tests for packing/unpacking tables --- include/pernix/detail/bits.h | 15 ++++ tests/pack_tables_tests.cpp | 154 ++++++++++++++++++++++++++++++++++ tests/unpack_tables_tests.cpp | 93 ++++++++++++++++++++ 3 files changed, 262 insertions(+) create mode 100644 include/pernix/detail/bits.h create mode 100644 tests/pack_tables_tests.cpp create mode 100644 tests/unpack_tables_tests.cpp diff --git a/include/pernix/detail/bits.h b/include/pernix/detail/bits.h new file mode 100644 index 0000000..c9f87cf --- /dev/null +++ b/include/pernix/detail/bits.h @@ -0,0 +1,15 @@ +#ifndef PERNIX_DETAIL_BITS_H +#define PERNIX_DETAIL_BITS_H + +#include + +namespace pernix::detail { +template +constexpr u32 low_bit_mask() { + static_assert(BitWidth >= 1, "low_bit_mask requires at least one bit"); + static_assert(BitWidth < 32, "low_bit_mask requires fewer than 32 bits"); + return (u32{1} << BitWidth) - 1u; +} +} // namespace pernix::detail + +#endif // PERNIX_DETAIL_BITS_H diff --git a/tests/pack_tables_tests.cpp b/tests/pack_tables_tests.cpp new file mode 100644 index 0000000..c0ddabe --- /dev/null +++ b/tests/pack_tables_tests.cpp @@ -0,0 +1,154 @@ +#include +#include + +#include +#include +#include + +#include + +namespace { +static_assert(pernix::detail::low_bit_mask<1>() == 0x00000001U); +static_assert(pernix::detail::low_bit_mask<7>() == 0x0000007fU); +static_assert(pernix::detail::low_bit_mask<8>() == 0x000000ffU); +static_assert(pernix::detail::low_bit_mask<9>() == 0x000001ffU); +static_assert(pernix::detail::low_bit_mask<15>() == 0x00007fffU); +static_assert(pernix::detail::low_bit_mask<16>() == 0x0000ffffU); +static_assert(pernix::detail::low_bit_mask<17>() == 0x0001ffffU); +static_assert(pernix::detail::low_bit_mask<24>() == 0x00ffffffU); + +template +constexpr bool validate_pack_table() { + using table = pernix::detail::pack_table; + using unsigned_type = std::make_unsigned_t; + + constexpr usize elements = VectorBytes / sizeof(LaneType); + constexpr usize lane_bits = sizeof(LaneType) * 8; + constexpr unsigned_type value_mask = (unsigned_type{1} << BitWidth) - 1; + constexpr unsigned_type lane_mask = static_cast(~unsigned_type{0}); + + std::array, table::contributor_count> reference_primary_permutes{}; + std::array, table::contributor_count> reference_primary_shifts{}; + std::array reference_primary_masks{}; + std::array reference_spill_permute{}; + std::array reference_spill_shift{}; + reference_spill_permute.fill(static_cast(-1)); + reference_spill_shift.fill(static_cast(lane_bits)); + for (usize contributor = 0; contributor < table::contributor_count; ++contributor) { + reference_primary_permutes[contributor].fill(static_cast(-1)); + reference_primary_shifts[contributor].fill(static_cast(lane_bits)); + } + + u64 reference_spill_mask = 0; + for (usize output_lane = 0; output_lane < elements; ++output_lane) { + const usize output_start = output_lane * lane_bits; + usize contributor = 0; + usize previous_input = elements; + + // Derive the plan independently from each packed output bit rather than interval arithmetic. + for (usize output_bit = 0; output_bit < lane_bits; ++output_bit) { + const usize packed_bit = output_start + output_bit; + const usize input_lane = packed_bit / BitWidth; + if (input_lane >= elements || input_lane == previous_input) { + continue; + } + previous_input = input_lane; + const auto shift = static_cast(input_lane * BitWidth) - static_cast(output_start); + if (shift < 0) { + reference_spill_permute[output_lane] = static_cast(input_lane); + reference_spill_shift[output_lane] = static_cast(-shift); + reference_spill_mask |= u64{1} << output_lane; + } else { + if (contributor >= table::contributor_count) { + return false; + } + reference_primary_permutes[contributor][output_lane] = static_cast(input_lane); + reference_primary_shifts[contributor][output_lane] = static_cast(shift); + reference_primary_masks[contributor] |= u64{1} << output_lane; + ++contributor; + } + } + } + + if (table::primary_permutes != reference_primary_permutes || table::primary_shifts != reference_primary_shifts || + table::primary_masks != reference_primary_masks || table::spill_permute != reference_spill_permute || + table::spill_shift != reference_spill_shift || table::spill_mask != reference_spill_mask) { + return false; + } + + std::array input{}; + for (usize lane = 0; lane < elements; ++lane) { + input[lane] = static_cast(((lane * 0x9e37U) + 0x5a5aU) & value_mask); + } + + for (usize output_lane = 0; output_lane < elements; ++output_lane) { + unsigned_type generated = 0; + const auto add_left = [&](const LaneType index, const LaneType shift) { + if (index >= 0) { + generated |= static_cast((input[static_cast(index)] << shift) & lane_mask); + } + }; + for (usize slot = 0; slot < table::contributor_count; ++slot) { + add_left(table::primary_permutes[slot][output_lane], table::primary_shifts[slot][output_lane]); + } + if (table::spill_permute[output_lane] >= 0) { + generated |= + static_cast(input[static_cast(table::spill_permute[output_lane])] >> table::spill_shift[output_lane]); + } + + unsigned_type expected = 0; + for (usize bit = 0; bit < lane_bits; ++bit) { + const usize packed_bit = (output_lane * lane_bits) + bit; + const usize input_lane = packed_bit / BitWidth; + const usize input_bit = packed_bit % BitWidth; + if (input_lane < elements) { + expected |= static_cast(((input[input_lane] >> input_bit) & 1U) << bit); + } + } + if (generated != expected) { + return false; + } + + const auto mask_bit = u64{1} << output_lane; + for (usize slot = 0; slot < table::contributor_count; ++slot) { + if (((table::primary_masks[slot] & mask_bit) != 0) != (table::primary_permutes[slot][output_lane] >= 0)) { + return false; + } + } + if (((table::spill_mask & mask_bit) != 0) != (table::spill_permute[output_lane] >= 0)) { + return false; + } + } + + return true; +} + +template +constexpr bool validate_8_bit_tables(std::index_sequence) { + return (validate_pack_table(Indices + 1), VectorBytes>() && ...); +} + +template +constexpr bool validate_16_bit_tables(std::index_sequence) { + return (validate_pack_table(Indices + 9), VectorBytes>() && ...); +} + +template +constexpr bool validate_32_bit_tables(std::index_sequence) { + return (validate_pack_table(Indices + 17), VectorBytes>() && ...); +} + +static_assert(validate_8_bit_tables<16>(std::make_index_sequence<8>{})); +static_assert(validate_8_bit_tables<32>(std::make_index_sequence<8>{})); +static_assert(validate_8_bit_tables<64>(std::make_index_sequence<8>{})); +static_assert(validate_16_bit_tables<16>(std::make_index_sequence<8>{})); +static_assert(validate_16_bit_tables<32>(std::make_index_sequence<8>{})); +static_assert(validate_16_bit_tables<64>(std::make_index_sequence<8>{})); +static_assert(validate_32_bit_tables<16>(std::make_index_sequence<8>{})); +static_assert(validate_32_bit_tables<32>(std::make_index_sequence<8>{})); +static_assert(validate_32_bit_tables<64>(std::make_index_sequence<8>{})); +} // namespace + +TEST(PackTablesTest, GeneratesAllAvx512LanePlansAtCompileTime) { + SUCCEED(); +} diff --git a/tests/unpack_tables_tests.cpp b/tests/unpack_tables_tests.cpp new file mode 100644 index 0000000..b7f3db6 --- /dev/null +++ b/tests/unpack_tables_tests.cpp @@ -0,0 +1,93 @@ +#include + +#include +#include + +#include + +namespace { +template +using shift_type_for = std::conditional_t<(BitWidth <= 8), i8, std::conditional_t<(BitWidth <= 16), i16, i32>>; + +template +constexpr bool validate_unpack_table() { + using table = pernix::detail::unpack_table; + + constexpr usize lane_bytes = sizeof(ShiftType); + constexpr usize lanes = VectorBytes / lane_bytes; + bool const expected_has_spill = false; + + if (table::primary_permute.size() != VectorBytes || table::right_shift.size() != lanes || + table::right_shift_magnitude.size() != lanes) { + return false; + } + + for (usize entry = 0; entry < lanes; ++entry) { + const usize bit_start = entry * BitWidth; + const usize first_byte = bit_start / 8; + const usize bit_offset = bit_start % 8; + const usize base = entry * lane_bytes; + + for (usize byte = 0; byte < lane_bytes; ++byte) { + if (table::primary_permute[base + byte] != first_byte + byte) { + return false; + } + } + if (table::right_shift[entry] != -static_cast(bit_offset)) { + return false; + } + if (table::right_shift_magnitude[entry] != static_cast(bit_offset)) { + return false; + } + + if constexpr (BitWidth <= 16) { + const bool spills = bit_offset + BitWidth > lane_bytes * 8; + expected_has_spill |= spills; + for (usize byte = 0; byte < lane_bytes; ++byte) { + const u8 expected = spills && byte == 0 ? static_cast(first_byte + lane_bytes) : pernix::detail::inactive_lane; + if (table::spill_permute[base + byte] != expected) { + return false; + } + } + + const ShiftType expected_shift = spills ? static_cast((lane_bytes * 8) - bit_offset) : 0; + if (table::left_shift_for_spill[entry] != expected_shift) { + return false; + } + } + } + + if constexpr (BitWidth <= 16) { + if (table::has_spill != expected_has_spill) { + return false; + } + } + + return true; +} + +template +constexpr bool validate_all_bit_widths(std::index_sequence) { + return (validate_unpack_table(Indices + 1)>, static_cast(Indices + 1), VectorBytes>() && ...); +} + +static_assert(validate_all_bit_widths<16>(std::make_index_sequence<24>{})); +static_assert(validate_all_bit_widths<32>(std::make_index_sequence<24>{})); +static_assert(validate_all_bit_widths<64>(std::make_index_sequence<24>{})); +} // namespace + +TEST(UnpackTablesTest, PreservesSignedShiftCountsForNeonCompatibleConsumers) { + using byte_table = pernix::detail::unpack_table; + EXPECT_EQ(byte_table::right_shift[1], -5); + EXPECT_EQ(byte_table::right_shift_magnitude[1], 5); + EXPECT_EQ(byte_table::left_shift_for_spill[1], 3); + + using halfword_table = pernix::detail::unpack_table; + EXPECT_EQ(halfword_table::right_shift[1], -5); + EXPECT_EQ(halfword_table::right_shift_magnitude[1], 5); + EXPECT_EQ(halfword_table::left_shift_for_spill[1], 11); + + using word_table = pernix::detail::unpack_table; + EXPECT_EQ(word_table::right_shift[1], -3); + EXPECT_EQ(word_table::right_shift_magnitude[1], 3); +} From 2bb954270701218f0d4dd766aaa4b56067c80f4d Mon Sep 17 00:00:00 2001 From: Felix Sternberg Date: Thu, 16 Jul 2026 20:49:46 +0200 Subject: [PATCH 43/43] refactor: update type definitions to use std:: types for consistency and clarity --- include/pernix/backend.hpp | 2 +- include/pernix/compat.h | 28 +++++++++++++++------------- include/pernix/pernix.h | 12 ++++++------ 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/include/pernix/backend.hpp b/include/pernix/backend.hpp index 228b1d1..9c3892d 100644 --- a/include/pernix/backend.hpp +++ b/include/pernix/backend.hpp @@ -4,7 +4,7 @@ #include namespace pernix { -enum class Backend { +enum class Backend : u8 { Auto = PERNIX_BACKEND_AUTO, Fallback = PERNIX_BACKEND_FALLBACK, FallbackScalar = PERNIX_BACKEND_FALLBACK_SCALAR, diff --git a/include/pernix/compat.h b/include/pernix/compat.h index ca3d10e..fc79a03 100644 --- a/include/pernix/compat.h +++ b/include/pernix/compat.h @@ -1,10 +1,12 @@ #ifndef PERNIX_COMPAT_H #define PERNIX_COMPAT_H -#if defined(__cplusplus) +#ifdef __cplusplus +#include #include #include #else +#include #include #include #endif @@ -30,20 +32,20 @@ #endif // Convenient type declarations -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; -typedef uintptr_t uptr; +using u8 = std::uint8_t; +using u16 = std::uint16_t; +using u32 = std::uint32_t; +using u64 = std::uint64_t; +using uptr = std::uintptr_t; -typedef int8_t i8; -typedef int16_t i16; -typedef int32_t i32; -typedef int64_t i64; +using i8 = std::int8_t; +using i16 = std::int16_t; +using i32 = std::int32_t; +using i64 = std::int64_t; -typedef float f32; -typedef double f64; +using f32 = std::float_t; +using f64 = std::double_t; -typedef std::size_t usize; +using usize = std::size_t; #endif // PERNIX_COMPAT_H diff --git a/include/pernix/pernix.h b/include/pernix/pernix.h index 2609f7d..407b3b6 100644 --- a/include/pernix/pernix.h +++ b/include/pernix/pernix.h @@ -3,15 +3,15 @@ #include -#if !defined(__cplusplus) +#ifndef __cplusplus #include #endif -#if defined(__cplusplus) +#ifdef __cplusplus extern "C" { #endif -typedef enum pernix_status { +typedef enum pernix_status { // NOLINT(cppcoreguidelines-use-enum-class, modernize-use-using) PERNIX_STATUS_OK = 0, PERNIX_STATUS_INVALID_ARGUMENT = -1, PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH = -2, @@ -20,7 +20,7 @@ typedef enum pernix_status { PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION = -5, } pernix_status; -typedef enum pernix_backend { +typedef enum pernix_backend { // NOLINT(cppcoreguidelines-use-enum-class, modernize-use-using) PERNIX_BACKEND_AUTO = 0, PERNIX_BACKEND_FALLBACK = 1, PERNIX_BACKEND_FALLBACK_SCALAR = PERNIX_BACKEND_FALLBACK, @@ -30,7 +30,7 @@ typedef enum pernix_backend { PERNIX_BACKEND_ARM64_NEON = 5, PERNIX_BACKEND_ARM64_SVE = 6, PERNIX_BACKEND_FALLBACK_STDPAR = 7, - PERNIX_BACKEND_FALLBACK_SIMD = 8 + PERNIX_BACKEND_FALLBACK_SIMD = 8, } pernix_backend; PERNIX_API u8 pernix_min_bit_width(void); @@ -95,7 +95,7 @@ PERNIX_API pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 PERNIX_API pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, void* output, u32 blocks, bool sign_values); -#if defined(__cplusplus) +#ifdef __cplusplus } #endif