diff --git a/.clang-format b/.clang-format index 3c83445..69ed367 100644 --- a/.clang-format +++ b/.clang-format @@ -1,11 +1,127 @@ +--- Language: Cpp BasedOnStyle: Google -AccessModifierOffset: -4 -AlignConsecutiveAssignments: true -AllowShortFunctionsOnASingleLine: InlineOnly +# General layout ColumnLimit: 140 +IndentWidth: 4 +ContinuationIndentWidth: 4 +TabWidth: 4 +UseTab: Never + +# Braces and compact constructs +BreakBeforeBraces: Attach +AllowShortBlocksOnASingleLine: Empty +AllowShortCaseExpressionOnASingleLine: true +AllowShortEnumsOnASingleLine: false +AllowShortFunctionsOnASingleLine: InlineOnly +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AllowShortLambdasOnASingleLine: Inline + +# Alignment +AlignAfterOpenBracket: true +AlignArrayOfStructures: Left +AlignConsecutiveAssignments: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false +AlignConsecutiveDeclarations: + Enabled: false +AlignConsecutiveMacros: + Enabled: false +AlignOperands: Align +AlignTrailingComments: + Kind: Always + OverEmptyLines: 0 + +# Pointers and references DerivePointerAlignment: false +PointerAlignment: Left +ReferenceAlignment: Pointer + +# Function declarations and calls +BinPackArguments: true +BinPackParameters: BinPack +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +PenaltyBreakBeforeFirstCallParameter: 19 + +# Templates +AlwaysBreakTemplateDeclarations: MultiLine +BreakTemplateDeclarations: Yes + +# Constructor initializers +BreakConstructorInitializers: BeforeColon +ConstructorInitializerIndentWidth: 4 +PackConstructorInitializers: BinPack + +# Operators and expressions +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakStringLiterals: true +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpacesInAngles: Never +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false + +# Includes +SortIncludes: CaseSensitive +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^' + Priority: 2 + SortPriority: 2 + + - Regex: '^<' + Priority: 3 + SortPriority: 3 + + - Regex: '^"' + Priority: 4 + SortPriority: 4 + + +# Namespaces +NamespaceIndentation: None FixNamespaceComments: true -IndentWidth: 4 -PointerAlignment: Left \ No newline at end of file +CompactNamespaces: false +ShortNamespaceLines: 1 + +# Access modifiers +AccessModifierOffset: -4 +EmptyLineBeforeAccessModifier: LogicalBlock +EmptyLineAfterAccessModifier: Never + +# Preprocessor +IndentPPDirectives: None +PPIndentWidth: 4 +IndentCaseLabels: true +IndentCaseBlocks: false + +# Comments +ReflowComments: Always +SpacesBeforeTrailingComments: 2 + +# Miscellaneous +InsertBraces: false +InsertNewlineAtEOF: true +KeepEmptyLines: + AtEndOfFile: false + AtStartOfBlock: false + AtStartOfFile: false +MaxEmptyLinesToKeep: 1 +SeparateDefinitionBlocks: Always +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope \ No newline at end of file diff --git a/.clang-tidy b/.clang-tidy index fe218fe..20ea7a3 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,13 +1,102 @@ +--- Checks: > - bugprone-* - cppcoreguidelines-* - clang-analyzer-* - modernize-* + -*, + clang-analyzer-*, + bugprone-*, + performance-*, + portability-*, + misc-*, + modernize-*, + readability-*, + cppcoreguidelines-*, + + -bugprone-easily-swappable-parameters, + -bugprone-narrowing-conversions, + -bugprone-signed-char-misuse, + + -cppcoreguidelines-avoid-c-arrays, + -cppcoreguidelines-avoid-do-while, + -cppcoreguidelines-avoid-magic-numbers, + -cppcoreguidelines-avoid-non-const-global-variables, + -cppcoreguidelines-init-variables, + -cppcoreguidelines-macro-usage, + -cppcoreguidelines-non-private-member-variables-in-classes-owning-memory, + -cppcoreguidelines-pro-bounds-constant-array-index, + -cppcoreguidelines-pro-bounds-pointer-arithmetic, + -cppcoreguidelines-pro-type-cstyle-cast, + -cppcoreguidelines-pro-type-reinterpret-cast, + -cppcoreguidelines-pro-type-union-access, + + -misc-include-cleaner, + -misc-no-recursion, + + -modernize-avoid-c-arrays, + -modernize-concat-nested-namespaces, + -modernize-loop-convert, + -modernize-return-braced-init-list, + -modernize-use-auto, + -modernize-use-nodiscard, + -modernize-use-trailing-return-type, + + -performance-avoid-endl, + + -readability-avoid-const-params-in-decls, + -readability-braces-around-statements, + -readability-convert-member-functions-to-static, + -readability-else-after-return, + -readability-function-cognitive-complexity, + -readability-function-size, + -readability-identifier-length, + -readability-identifier-naming, + -readability-implicit-bool-conversion, + -readability-isolate-declaration, + -readability-magic-numbers, + -readability-make-member-function-const, + -readability-named-parameter, + -readability-redundant-access-specifiers, + -readability-static-accessed-through-instance + +WarningsAsErrors: > + clang-analyzer-*, + bugprone-*, performance-* - readability-* - misc-* - -modernize-use-trailing-return-type -WarningsAsErrors: 'bugprone-*,clang-analyzer-*' -HeaderFilterRegex: '' -FormatStyle: none \ No newline at end of file +HeaderFilterRegex: '^(include|src|tests|benchmarks)/' +ExcludeHeaderFilterRegex: > + (^|/)(build|cmake-build-[^/]*|third_party|external|vendor|generated)/ + +SystemHeaders: false +FormatStyle: file +UseColor: true + +CheckOptions: + # Suspiciously adjacent parameters are common in low-level SIMD helpers. + bugprone-easily-swappable-parameters.MinimumLength: 3 + bugprone-easily-swappable-parameters.IgnoredParameterNames: '"",begin,end,first,last,lhs,rhs,x,y,z' + + # Treat assertions as side-effecting expressions. + bugprone-assert-side-effect.AssertMacros: 'assert,PERNIX_ASSERT' + + # Permit common test and benchmark patterns. + bugprone-unchecked-optional-access.CheckOsLinuxAndMac: true + + # Do not rewrite auto when it improves readability or hides intrinsic types. + modernize-use-auto.MinTypeNameLength: 12 + modernize-use-auto.RemoveStars: false + + # Allow compact SIMD and mathematical variable names. + readability-identifier-length.MinimumVariableNameLength: 1 + readability-identifier-length.MinimumParameterNameLength: 1 + readability-identifier-length.IgnoredVariableNames: '^[ijknxyz]|lo|hi|src|dst|tmp|mask|count|scale$' + readability-identifier-length.IgnoredParameterNames: '^[ijknxyz]|lo|hi|src|dst|input|output|count|scale$' + + # Keep function complexity checks informative if enabled manually. + readability-function-cognitive-complexity.Threshold: 40 + readability-function-cognitive-complexity.DescribeBasicIncrements: true + + # Ignore conventional bit-oriented numeric literals. + readability-magic-numbers.IgnoredIntegerValues: > + 0;1;2;3;4;7;8;15;16;24;31;32;63;64;127;128;255;256;511;512 + readability-magic-numbers.IgnorePowersOf2IntegerValues: true + readability-magic-numbers.IgnoreTypeAliases: true + readability-magic-numbers.IgnoreBitFieldsWidths: true \ No newline at end of file diff --git a/.clangd b/.clangd new file mode 100644 index 0000000..368ac08 --- /dev/null +++ b/.clangd @@ -0,0 +1,69 @@ +--- +CompileFlags: + CompilationDatabase: build + Add: + - -Wall + - -Wextra + - -Wpedantic + - -Wconversion + - -Wsign-conversion + - -Wshadow + - -Wundef + - -Wdouble-promotion + - -Wformat=2 + - -Wimplicit-fallthrough + - -Wmissing-declarations + - -Wmissing-prototypes + - -Wnon-virtual-dtor + - -Wold-style-cast + - -Woverloaded-virtual + - -Wzero-as-null-pointer-constant + Remove: + - -Werror + - -Wno-* + +Diagnostics: + Suppress: + - pp_including_mainfile_in_preamble + + ClangTidy: + Add: + - clang-analyzer-* + - bugprone-* + - performance-* + - portability-* + + Remove: + - bugprone-easily-swappable-parameters + - bugprone-narrowing-conversions + - bugprone-signed-char-misuse + + FastCheckFilter: Strict + + UnusedIncludes: Strict + MissingIncludes: Strict + +Index: + Background: Build + StandardLibrary: true + +Completion: + AllScopes: true + ArgumentLists: FullPlaceholders + HeaderInsertion: IWYU + CodePatterns: All + +Hover: + ShowAKA: true + +InlayHints: + Enabled: true + ParameterNames: true + DeducedTypes: true + Designators: true + BlockEnd: false + TypeNameLimit: 32 + +SemanticTokens: + DisabledKinds: [] + DisabledModifiers: [] \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f1547cd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,301 @@ +name: CI + +on: + push: + branches: + - main + - "release/**" + tags: + - "v*" + pull_request: + branches: + - main + - "release/**" + +env: + CCACHE_DIR: /home/runner/.cache/ccache + CCACHE_MAXSIZE: 500M + +jobs: + release: + name: ${{ matrix.compiler }} release + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + compiler: [gcc, clang] + include: + - compiler: gcc + cc: gcc + cxx: g++ + - compiler: clang + cc: clang + cxx: clang++ + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ clang + + - name: Cache ccache + uses: actions/cache@v6 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-${{ matrix.compiler }}-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-${{ matrix.compiler }}-${{ github.job }}- + ccache-${{ runner.os }}-${{ matrix.compiler }}- + ccache-${{ runner.os }}- + + - name: Configure + run: | + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=${{ matrix.cc }} \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_INSTALL=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF + + - name: Build + run: cmake --build build --config Release -j"$(nproc)" + + - name: Test + run: ctest --test-dir build --output-on-failure + + - name: Run examples + run: | + ./build/examples/pernix_example_cpp + ./build/examples/pernix_example_c + + arm64-release: + name: arm64 release + runs-on: ubuntu-24.04-arm + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ + + - name: Show runner CPU + run: | + uname -m + lscpu + grep -m1 Features /proc/cpuinfo || true + + - name: Cache ccache + uses: actions/cache@v6 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-arm64-gcc-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-arm64-gcc-${{ github.job }}- + ccache-${{ runner.os }}-arm64-gcc- + ccache-${{ runner.os }}-arm64- + ccache-${{ runner.os }}- + + - name: Configure + run: | + cmake -S . -B build-arm64 -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_INSTALL=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build + run: cmake --build build-arm64 --config Release -j"$(nproc)" + + - name: Test + run: ctest --test-dir build-arm64 --output-on-failure + + - name: Run examples + run: | + ./build-arm64/examples/pernix_example_cpp + ./build-arm64/examples/pernix_example_c + + fallback-only: + name: fallback-only release + runs-on: ubuntu-latest + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ + + - name: Cache ccache + uses: actions/cache@v6 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-gcc-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-gcc-${{ github.job }}- + ccache-${{ runner.os }}-gcc- + ccache-${{ runner.os }}- + + - name: Configure + run: | + cmake -S . -B build-fallback -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_INSTALL=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_ENABLE_X86_AVX2=OFF \ + -DPERNIX_ENABLE_X86_BMI2=OFF \ + -DPERNIX_ENABLE_X86_AVX512VBMI=OFF \ + -DPERNIX_ENABLE_ARM64_NEON=OFF \ + -DPERNIX_ENABLE_ARM64_SVE2=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build + run: cmake --build build-fallback --config Release -j"$(nproc)" + + - name: Test + run: ctest --test-dir build-fallback --output-on-failure + + - name: Run examples + run: | + ./build-fallback/examples/pernix_example_cpp + ./build-fallback/examples/pernix_example_c + + debug-and-sanitizers: + name: debug and sanitizers + runs-on: ubuntu-latest + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ + + - name: Cache ccache + uses: actions/cache@v6 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-gcc-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-gcc-${{ github.job }}- + ccache-${{ runner.os }}-gcc- + ccache-${{ runner.os }}- + + - name: Configure debug + run: | + cmake -S . -B build-debug -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_INSTALL=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_ENABLE_X86_AVX2=OFF \ + -DPERNIX_ENABLE_X86_BMI2=OFF \ + -DPERNIX_ENABLE_X86_AVX512VBMI=OFF \ + -DPERNIX_ENABLE_ARM64_NEON=OFF \ + -DPERNIX_ENABLE_ARM64_SVE2=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build debug + run: cmake --build build-debug --config Debug -j"$(nproc)" + + - name: Test debug + run: ctest --test-dir build-debug --output-on-failure + + - name: Configure sanitizers + run: | + cmake -S . -B build-sanitizers -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=OFF \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=OFF \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_ENABLE_SANITIZER_ADDRESS=ON \ + -DPERNIX_ENABLE_SANITIZER_UNDEFINED=ON \ + -DPERNIX_ENABLE_X86_AVX2=OFF \ + -DPERNIX_ENABLE_X86_BMI2=OFF \ + -DPERNIX_ENABLE_X86_AVX512VBMI=OFF \ + -DPERNIX_ENABLE_ARM64_NEON=OFF \ + -DPERNIX_ENABLE_ARM64_SVE2=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build sanitizers + run: cmake --build build-sanitizers --config Debug -j"$(nproc)" + + - name: Test sanitizers + run: ctest --test-dir build-sanitizers --output-on-failure + + fortran: + name: Fortran bindings + runs-on: ubuntu-latest + + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libgtest-dev ccache gcc g++ gfortran + + - name: Cache ccache + uses: actions/cache@v6 + with: + path: /home/runner/.cache/ccache + key: ccache-${{ runner.os }}-gfortran-${{ github.job }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'src/**', 'include/**', 'tests/**', 'examples/**', 'bindings/**') }} + restore-keys: | + ccache-${{ runner.os }}-gfortran-${{ github.job }}- + ccache-${{ runner.os }}-gfortran- + ccache-${{ runner.os }}- + + - name: Configure + run: | + cmake -S . -B build-fortran -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_Fortran_COMPILER_LAUNCHER=ccache \ + -DPERNIX_ENABLE_TESTS=ON \ + -DPERNIX_ENABLE_EXAMPLES=ON \ + -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON \ + -DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=OFF \ + -DPERNIX_ENABLE_FALLBACK_STDPAR=OFF \ + -DPERNIX_TEST_BLOCK_SIZES=64 + + - name: Build + run: cmake --build build-fortran --config Release -j"$(nproc)" + + - name: Test + run: ctest --test-dir build-fortran --output-on-failure + + - name: Run Fortran example + run: ./build-fortran/examples/pernix_example_fortran diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index ae538ec..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "external/simde"] - path = external/simde - url = https://github.com/simd-everywhere/simde diff --git a/CMakeLists.txt b/CMakeLists.txt index edf6c78..da59443 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,25 +1,100 @@ -cmake_minimum_required(VERSION 3.30) +cmake_minimum_required(VERSION 3.25) project(pernix VERSION 0.1.0 LANGUAGES CXX C) -option(PERNIX_ENABLE_TESTS "Enable tests for pernix" on) -option(PERNIX_ENABLE_CODE_COVERAGE "Enable code coverage for pernix (requires lcov/genhtml)" off) -option(PERNIX_ENABLE_LTO "Enable Link Time Optimization (LTO)" off) -option(PERNIX_ENABLE_INSTALL "Enable installation of pernix" on) -option(PERNIX_ENABLE_DOXYGEN "Build documentation with Doxygen." off) -option(PERNIX_INSTALL_DOCS "Install documentation with pernix" off) +option(BUILD_SHARED_LIBS "Build pernix as a shared library by default" ON) +option(PERNIX_ENABLE_TESTS "Build pernix unit and integration tests" ON) +option(PERNIX_ENABLE_CODE_COVERAGE "Enable code coverage for pernix tests (requires lcov/genhtml)" OFF) +option(PERNIX_ENABLE_LTO "Enable interprocedural optimization/link-time optimization" OFF) +option(PERNIX_ENABLE_SANITIZER_ADDRESS "Build pernix targets with AddressSanitizer instrumentation" OFF) +option(PERNIX_ENABLE_SANITIZER_UNDEFINED "Build pernix targets with UndefinedBehaviorSanitizer instrumentation" OFF) +option(PERNIX_ENABLE_INSTALL "Install pernix library, headers, and package metadata" ON) +option(PERNIX_ENABLE_DOXYGEN "Build API documentation with Doxygen" OFF) +option(PERNIX_INSTALL_DOCS "Install generated Doxygen documentation; requires PERNIX_ENABLE_DOXYGEN=ON" OFF) +option(PERNIX_ENABLE_EXAMPLES "Build pernix example programs" OFF) +option(PERNIX_ENABLE_INSTALL_CONSUMER_TESTS "Add CTest checks for consuming an installed pernix package" ON) + +option(PERNIX_ENABLE_X86_BMI2 "Build x86 BMI2 backend when targeting x86" ON) +option(PERNIX_ENABLE_X86_AVX2 "Build x86 AVX2 backend when targeting x86" ON) +option(PERNIX_ENABLE_X86_AVX512VBMI "Build x86 AVX512-VBMI backend when targeting x86" ON) +option(PERNIX_ENABLE_ARM64_NEON "Build arm64 NEON backend when targeting arm64" ON) +option(PERNIX_ENABLE_ARM64_SVE2 "Build arm64 SVE2 backend when targeting arm64" ON) +set(PERNIX_ENABLE_FALLBACK_STDPAR "OFF" CACHE STRING "Enable the stdpar fallback backend: OFF, AUTO, or ON. TBB is optional and enables parallel standard execution policies when found.") +set_property(CACHE PERNIX_ENABLE_FALLBACK_STDPAR PROPERTY STRINGS AUTO ON OFF) +set(PERNIX_ENABLE_FALLBACK_SIMD "AUTO" CACHE STRING "Enable the C++26 std::simd fallback backend: OFF, AUTO, or ON.") +set_property(CACHE PERNIX_ENABLE_FALLBACK_SIMD PROPERTY STRINGS AUTO ON OFF) + +option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" OFF) + +option(PERNIX_ENABLE_FORTRAN_BINDINGS "Build optional Fortran bindings and round-trip test" OFF) -option(PERNIX_DISABLE_BMI2 "Disable BMI2 optimizations" off) -option(PERNIX_DISABLE_AVX2 "Disable AVX2 optimizations" off) -option(PERNIX_DISABLE_AVX512 "Disable AVX512 optimizations" off) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") +set(PERNIX_BUNDLE_SIMDE_FOR_INSTALL OFF) -option(PERNIX_USE_SIMDE "Use SIMDe library for portable SIMD support" off) +string(TOUPPER "${PERNIX_ENABLE_FALLBACK_STDPAR}" PERNIX_ENABLE_FALLBACK_STDPAR) +set(PERNIX_ENABLE_FALLBACK_STDPAR "${PERNIX_ENABLE_FALLBACK_STDPAR}" CACHE STRING "Enable the stdpar fallback backend: OFF, AUTO, or ON. TBB is optional and enables parallel standard execution policies when found." FORCE) +if (NOT PERNIX_ENABLE_FALLBACK_STDPAR MATCHES "^(AUTO|ON|OFF)$") + message(FATAL_ERROR "PERNIX_ENABLE_FALLBACK_STDPAR must be AUTO, ON, or OFF") +endif () -option(PERNIX_ENABLE_FORTRAN_BINDINGS "Build Fortran bindings for pernix" off) +string(TOUPPER "${PERNIX_ENABLE_FALLBACK_SIMD}" PERNIX_ENABLE_FALLBACK_SIMD) +set(PERNIX_ENABLE_FALLBACK_SIMD "${PERNIX_ENABLE_FALLBACK_SIMD}" CACHE STRING "Enable the C++26 std::simd fallback backend: OFF, AUTO, or ON." FORCE) +if (NOT PERNIX_ENABLE_FALLBACK_SIMD MATCHES "^(AUTO|ON|OFF)$") + message(FATAL_ERROR "PERNIX_ENABLE_FALLBACK_SIMD must be AUTO, ON, or OFF") +endif () -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") +set(PERNIX_BUILD_FALLBACK_SIMD OFF) +if (NOT PERNIX_ENABLE_FALLBACK_SIMD STREQUAL "OFF") + include(CheckCXXSourceCompiles) + if (DEFINED CMAKE_CXX_STANDARD) + set(PERNIX_SAVED_CMAKE_CXX_STANDARD "${CMAKE_CXX_STANDARD}") + set(PERNIX_CXX_STANDARD_WAS_DEFINED ON) + endif () + if (DEFINED CMAKE_CXX_STANDARD_REQUIRED) + set(PERNIX_SAVED_CMAKE_CXX_STANDARD_REQUIRED "${CMAKE_CXX_STANDARD_REQUIRED}") + set(PERNIX_CXX_STANDARD_REQUIRED_WAS_DEFINED ON) + endif () + set(CMAKE_CXX_STANDARD 26) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + check_cxx_source_compiles([=[ + #include + #include + + #if !defined(__cpp_lib_simd) + #error "The standard library does not provide C++26 std::simd" + #endif + + int main() { + std::simd values{}; + return static_cast(values.size() == 0); + } + ]=] PERNIX_HAS_CXX26_STDSIMD) + if (PERNIX_CXX_STANDARD_WAS_DEFINED) + set(CMAKE_CXX_STANDARD "${PERNIX_SAVED_CMAKE_CXX_STANDARD}") + else () + unset(CMAKE_CXX_STANDARD) + endif () + if (PERNIX_CXX_STANDARD_REQUIRED_WAS_DEFINED) + set(CMAKE_CXX_STANDARD_REQUIRED "${PERNIX_SAVED_CMAKE_CXX_STANDARD_REQUIRED}") + else () + unset(CMAKE_CXX_STANDARD_REQUIRED) + endif () -if (PERNIX_USE_SIMDE) - add_subdirectory(external/simde EXCLUDE_FROM_ALL) + if (PERNIX_HAS_CXX26_STDSIMD) + set(PERNIX_BUILD_FALLBACK_SIMD ON) + message(STATUS "PERNIX fallback SIMD backend enabled (C++26 std::simd available)") + elseif (PERNIX_ENABLE_FALLBACK_SIMD STREQUAL "ON") + message(FATAL_ERROR "PERNIX_ENABLE_FALLBACK_SIMD=ON requires compiler and standard-library support for C++26 std::simd") + else () + message(STATUS "PERNIX fallback SIMD backend disabled (C++26 std::simd unavailable)") + endif () +endif () + +if (PERNIX_INSTALL_DOCS AND NOT PERNIX_ENABLE_DOXYGEN) + message(FATAL_ERROR "PERNIX_INSTALL_DOCS requires PERNIX_ENABLE_DOXYGEN=ON") +endif () + +if (PERNIX_ENABLE_INSTALL_CONSUMER_TESTS AND PERNIX_ENABLE_TESTS AND NOT PERNIX_ENABLE_INSTALL) + message(FATAL_ERROR "PERNIX_ENABLE_INSTALL_CONSUMER_TESTS requires PERNIX_ENABLE_INSTALL=ON") endif () include(CTest) @@ -40,61 +115,108 @@ else () endif () message(STATUS "Pernix version: ${VERSION}, normalized to ${NORMALIZED_VERSION}") -set(BENCHMARK_CXX_STANDARD 20) +if (NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|Intel|NVHPC") + message(FATAL_ERROR "Unsupported compiler: ${CMAKE_CXX_COMPILER_ID}") +endif () -set(CMAKE_CXX_STANDARD ${BENCHMARK_CXX_STANDARD}) -set(CMAKE_CXX_STANDARD_REQUIRED YES) -set(CMAKE_CXX_EXTENSIONS OFF) +include(CheckCXXCompilerFlag) +set(PERNIX_PRIVATE_COMPILE_OPTIONS) +foreach (PERNIX_CXX_FLAG + -Wall + -Wextra + -Wpedantic + -Wshadow + -Wfloat-equal + -Wold-style-cast + -Wconversion + -fstrict-aliasing + -Wno-ignored-attributes +) + string(MAKE_C_IDENTIFIER "PERNIX_HAS_CXX_FLAG_${PERNIX_CXX_FLAG}" PERNIX_CXX_FLAG_VARIABLE) + check_cxx_compiler_flag("${PERNIX_CXX_FLAG}" "${PERNIX_CXX_FLAG_VARIABLE}") + if (${PERNIX_CXX_FLAG_VARIABLE}) + list(APPEND PERNIX_PRIVATE_COMPILE_OPTIONS "${PERNIX_CXX_FLAG}") + else () + message(STATUS "Compiler flag not supported: ${PERNIX_CXX_FLAG}") + endif () +endforeach () -include(AddCXXCompilerFlag) -if (MSVC) - message(FATAL_ERROR "MSVC compiler is not supported") -else () - add_cxx_compiler_flag(-Wall) - add_cxx_compiler_flag(-Wextra) - add_cxx_compiler_flag(-Wshadow) - add_cxx_compiler_flag(-Wfloat-equal) - add_cxx_compiler_flag(-Wold-style-cast) - add_cxx_compiler_flag(-Wconversion) - add_cxx_compiler_flag(-fstrict-aliasing) - add_cxx_compiler_flag(-Wno-ignored-attributes) - - if (PERNIX_ENABLE_LTO) - add_cxx_compiler_flag(-flto=auto) - add_cxx_compiler_flag(-Wno-lto-type-mismatch) - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - find_program(GCC_AR gcc-ar) - if (GCC_AR) - set(CMAKE_AR ${GCC_AR}) - endif () - find_program(GCC_RANLIB gcc-ranlib) - if (GCC_RANLIB) - set(CMAKE_RANLIB ${GCC_RANLIB}) - endif () - elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") - find_program(LLVM_AR llvm-ar) - if (LLVM_AR) - set(CMAKE_AR ${LLVM_AR}) - endif () - find_program(LLVM_RANLIB llvm-ranlib) - if (LLVM_RANLIB) - set(CMAKE_RANLIB ${LLVM_RANLIB}) - endif () +set(PERNIX_SANITIZER_OPTIONS) +if (PERNIX_ENABLE_SANITIZER_ADDRESS OR PERNIX_ENABLE_SANITIZER_UNDEFINED) + if (NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|IntelLLVM") + message(FATAL_ERROR "PERNIX sanitizer options are supported only with GCC/Clang-like compilers") + endif () +endif () + +if (PERNIX_ENABLE_SANITIZER_ADDRESS) + list(APPEND PERNIX_SANITIZER_OPTIONS "-fsanitize=address") +endif () + +if (PERNIX_ENABLE_SANITIZER_UNDEFINED) + list(APPEND PERNIX_SANITIZER_OPTIONS "-fsanitize=undefined") +endif () + +if (PERNIX_SANITIZER_OPTIONS) + add_compile_options(${PERNIX_SANITIZER_OPTIONS}) + add_link_options(${PERNIX_SANITIZER_OPTIONS}) +endif () + +if (PERNIX_ENABLE_LTO) + include(CheckIPOSupported) + check_ipo_supported(RESULT PERNIX_IPO_SUPPORTED OUTPUT PERNIX_IPO_ERROR) + if (NOT PERNIX_IPO_SUPPORTED) + message(FATAL_ERROR "PERNIX_ENABLE_LTO is enabled, but IPO/LTO is not supported: ${PERNIX_IPO_ERROR}") + endif () + + check_cxx_compiler_flag("-Wno-lto-type-mismatch" PERNIX_HAS_CXX_FLAG_WNO_LTO_TYPE_MISMATCH) + if (PERNIX_HAS_CXX_FLAG_WNO_LTO_TYPE_MISMATCH) + list(APPEND PERNIX_PRIVATE_COMPILE_OPTIONS "-Wno-lto-type-mismatch") + endif () + + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + find_program(GCC_AR gcc-ar) + if (GCC_AR) + set(CMAKE_AR ${GCC_AR}) + endif () + find_program(GCC_RANLIB gcc-ranlib) + if (GCC_RANLIB) + set(CMAKE_RANLIB ${GCC_RANLIB}) + endif () + elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + find_program(LLVM_AR llvm-ar) + if (LLVM_AR) + set(CMAKE_AR ${LLVM_AR}) + endif () + find_program(LLVM_RANLIB llvm-ranlib) + if (LLVM_RANLIB) + set(CMAKE_RANLIB ${LLVM_RANLIB}) endif () endif () endif () -include_directories(${PROJECT_SOURCE_DIR}/include) +set(PERNIX_TARGET_IS_X86 OFF) +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|i[3-6]86|i686)$") + set(PERNIX_TARGET_IS_X86 ON) +endif () + +set(PERNIX_TARGET_IS_ARM64 OFF) +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") + set(PERNIX_TARGET_IS_ARM64 ON) +endif () add_subdirectory(src) if (PERNIX_ENABLE_FORTRAN_BINDINGS) message(STATUS "Building Fortran bindings for pernix") - include(CMakeAddFortranSubdirectory) - cmake_add_fortran_subdirectory(bindings/fortran NO_EXTERNAL_INSTALL) + enable_language(Fortran) + add_subdirectory(bindings/fortran) +endif () + +if (PERNIX_ENABLE_EXAMPLES) + add_subdirectory(examples) endif () if (PERNIX_ENABLE_TESTS) enable_testing() add_subdirectory(tests) -endif () \ No newline at end of file +endif () diff --git a/README.md b/README.md index 7c018d9..f95f5bd 100644 --- a/README.md +++ b/README.md @@ -1,125 +1,315 @@ -# PERNIX: Floating-Point Number De/Compression on CPUs -PERNIX is a high-throughput floating-point compression library for CPU-based scientific workloads. It quantizes floating-point values to a configurable bit width and packs them into fixed-size blocks, reducing memory and communication bandwidth while keeping decompression fast. +# PERNIX -The library provides: +PERNIX is a small C/C++ library for CPU-based scientific data compression. It quantizes `float` or `double` values to a +user-chosen bit width and packs the quantized integers into fixed 64-byte blocks. The main goal is fast block +decompression with a portable fallback path and optional SIMD backends. -* C++ template API (`pernix::compress_block`, `pernix::decompress_block`) -* C ABI wrappers (`compress_block`, `decompress_block`, and `_f64` variants) -* Fortran bindings in `bindings/fortran` -* SIMD-optimized backends (AVX2, AVX-512 VBMI, BMI2) with fallback implementations +The public API includes: -Compression of floating-point numbers $x_i$ to $N$-bit quantized numbers with scale $\varepsilon$ (M. Guidon, F. Schiffmann, J. Hutter and J. VandeVondele, The Journal of Chemical Physics, 2008, 128, 214104): +* C++ functions in ``, including `pernix::compress_block` and `pernix::decompress_block` +* a plain C ABI in ``, including `pernix_compress_block_f32` and `pernix_decompress_block_f32` +* `_f64` variants for double-precision input/output +* optional Fortran bindings that call the C ABI -$$b_{\max} = \max\left(\lvert x_i \rvert\right) \quad \quad \varepsilon = \frac{b_{\max}}{2^{N-1}-1}$$ +## Block Format -$$x_{i,N} = \mathrm{ANINT}\left(x_i\cdot\varepsilon^{-1}\right)$$ +The normal PERNIX block is exactly 64 bytes, or 512 bits. For bit width `N`, one block stores: -PERNIX is block-based and uses 64-byte (512-bit) compressed blocks by default. For a bit width `N`, each block stores `(64 * 8) / N` values. +```text +elements_per_block = (64 * 8) / N +``` + +Valid public bit widths are `1..24`. Public helpers expose these constants and calculations: + +* `pernix::compressed_block_size()` / `pernix_compressed_block_size()` returns `64` +* `pernix::elements_per_block(bit_width)` / `pernix_elements_per_block(bit_width)` returns the 64-byte element count +* `pernix::is_valid_bit_width(bit_width)` / `pernix_is_valid_bit_width(bit_width)` checks `1..24` +* `pernix::min_bit_width()` and `pernix::max_bit_width()` return `1` and `24` + +The implementation also accepts explicit block sizes `128`, `256`, `512`, and `1024` for internal/test coverage. The +documented interchange format is the 64-byte block. + +Packed values are written as a byte stream, least-significant bits first within each value and byte. Compression +zero-fills the output block before packing, so unused tail bits in a block are zero. When `sign_values=true`, +decompression +sign-extends each `N`-bit value before multiplying by scale. When `sign_values=false`, decompression treats the packed +value as unsigned. For `bit_width == 1`, signed fallback compression clamps to binary `0/1`. + +The scalar fallback packs and unpacks byte-by-byte, so the serialized 64-byte block is not a native integer dump. +Backends +are expected to produce compatible blocks for the same inputs, bit width, scale, and sign mode. + +## Quantization And Scale + +For a block with maximum magnitude `bmax` and bit width `N`, PERNIX uses this forward decompression scale: + +```text +scale = bmax / (2^(N - 1) - 1) +``` + +For `N == 1`, the denominator is treated as `1`. For `bmax == 0`, helper functions return a small positive scale rather +than zero. + +The current compression API expects the inverse scale: + +```text +quantized = round(input * inverse_scale) +``` + +The decompression API expects the forward scale: + +```text +output = quantized * scale +``` -## Compiling -1. clone repository `git clone https://github.com/pc2/pernix` -2. build with CMake: - * `cmake -E make_directory "build"` - * `cmake -E chdir "build" cmake -DCMAKE_BUILD_TYPE=Release -DPERNIX_ENABLE_TESTS=off ../` - * `cmake --build "build" --config Release` -3. `libpernix.so` will be in `build/src` +Use these helpers to avoid mixing the two conventions: -To enable Fortran bindings, configure with `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON`. +* C++: `pernix::decompression_scale_from_bmax`, `pernix::compression_scale_from_bmax`, `pernix::inverse_scale` +* C: `pernix_decompression_scale_f32`, `pernix_compression_scale_f32`, `pernix_inverse_scale_f32` +* C f64: the same names with `_f64` -## Usage Examples +Compatibility aliases `pernix::scale_from_bmax`, `pernix_scale_f32`, and `pernix_scale_f64` compute the forward +decompression scale. -### C++ API example (single block) +Scale arguments passed to compression and decompression must be finite and greater than zero. Zero, negative, NaN, or +infinite scales return `PERNIX_STATUS_INVALID_ARGUMENT`. + +## API Contracts + +All public compression and decompression calls return `pernix_status` (`pernix::Status` in C++). + +Status values: + +* `PERNIX_STATUS_OK`: success +* `PERNIX_STATUS_INVALID_ARGUMENT`: null pointer, invalid span size, zero block count, or invalid scale +* `PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH`: bit width outside `1..24` +* `PERNIX_STATUS_UNSUPPORTED_BACKEND`: backend enum value is unknown +* `PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE`: block size is not supported +* `PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION`: backend is not compiled in or is unavailable on this CPU + +Use `pernix::status_string(status)` or `pernix_status_string(status)` for readable names. + +For one block: + +* compression reads at least `(block_size * 8) / bit_width` `float` or `double` values +* compression writes exactly `block_size` bytes +* decompression reads exactly `block_size` bytes +* decompression writes at least `(block_size * 8) / bit_width` `float` or `double` values + +For `*_blocks` calls, multiply those sizes by `blocks`. `blocks == 0` is invalid. The C ABI validates null pointers and +basic parameters, but it cannot validate buffer lengths. The C++ `std::span` wrappers validate span sizes before calling +the lower-level kernels. + +The `_f32` APIs operate on `float` data and take `float` scale values. The `_f64` APIs operate on `double` data and take +`double` scale values. Both use the same packed integer block format for a given bit width. + +Given the same backend, inputs, bit width, scale, and sign mode, behavior is deterministic. Different backends are +tested +for compatible results, but exact floating-point details should not be treated as a cross-backend ABI guarantee beyond +the +documented quantization model. + +## Backends + +`PERNIX_BACKEND_FALLBACK` is the portable scalar backend and is always available. `PERNIX_BACKEND_AUTO` selects an +available optimized backend when one is compiled and supported on the host CPU, otherwise it falls back. + +Current backend enum values: + +* `PERNIX_BACKEND_AUTO` +* `PERNIX_BACKEND_FALLBACK` +* `PERNIX_BACKEND_X86_AVX2` +* `PERNIX_BACKEND_X86_BMI2` +* `PERNIX_BACKEND_X86_AVX512_VBMI` +* `PERNIX_BACKEND_ARM64_NEON` +* `PERNIX_BACKEND_ARM64_SVE` +* `PERNIX_BACKEND_FALLBACK_STDPAR` +* `PERNIX_BACKEND_FALLBACK_SIMD` + +x86 SIMD kernels are compiled with per-source ISA flags when enabled. Generic dispatch and fallback code are built for +the +baseline target. ARM decompression paths exist, but ARM compression is incomplete/stubbed. The stdpar fallback is +disabled +by default. When enabled, both the compiled library and the header-only target export stdpar dispatch. If TBB is +available, +stdpar uses standard parallel execution policies; otherwise it preserves the same 8-value grouping logic while executing +sequentially. + +## C++ Example ```cpp +#include + #include #include #include -#include -#include +#include int main() { - constexpr uint8_t BIT_WIDTH = 16; - constexpr uint32_t BLOCK_SIZE = 64; - constexpr size_t ELEMENTS = (BLOCK_SIZE * 8) / BIT_WIDTH; // 32 values for 16-bit + constexpr u8 bit_width = 16; + constexpr u32 block_size = pernix::compressed_block_size(); + constexpr usize elements = pernix::elements_per_block(bit_width); - std::array input{}; - for (size_t i = 0; i < ELEMENTS; ++i) { - input[i] = std::sin(static_cast(i)); + std::array input{}; + for (usize i = 0; i < input.size(); ++i) { + input[i] = std::sin(static_cast(i) * 0.25f); } float bmax = 0.0f; - for (float x : input) { - bmax = std::max(bmax, std::abs(x)); + for (float value : input) { + bmax = std::max(bmax, std::abs(value)); } - const float scale = bmax / ((1u << (BIT_WIDTH - 1)) - 1u); - - std::array compressed{}; - std::array restored{}; - if (pernix::compress_block(input.data(), scale, compressed.data()) != 0) { + float scale = 0.0f; + float inverse_scale = 0.0f; + if (pernix::decompression_scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK || + pernix::inverse_scale(scale, inverse_scale) != PERNIX_STATUS_OK) { return 1; } - if (pernix::decompress_block(compressed.data(), scale, restored.data()) != 0) { - return 1; + + std::array compressed{}; + std::array restored{}; + + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, inverse_scale, + compressed) != PERNIX_STATUS_OK) { + return 2; + } + if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, scale, + restored) != PERNIX_STATUS_OK) { + return 3; } - return 0; } ``` -### C ABI example (single block) +## C Example ```c -#include -#include #include int main(void) { - const uint8_t bit_width = 16; - float input[32]; - uint8_t compressed[64]; - float restored[32]; - float scale = 1.0f; - - for (int i = 0; i < 32; ++i) { - input[i] = sinf((float)i); + enum { bit_width = 16, block_size = 64, elements = (block_size * 8) / bit_width }; + float input[elements]; + float restored[elements]; + u8 compressed[block_size]; + float bmax = 0.0f; + + for (int i = 0; i < elements; ++i) { + input[i] = ((float)i - 16.0f) * 0.125f; + const float magnitude = input[i] < 0.0f ? -input[i] : input[i]; + bmax = bmax < magnitude ? magnitude : bmax; } - if (compress_block(bit_width, input, scale, compressed) != 0) { + float scale = 0.0f; + float inverse_scale = 0.0f; + if (pernix_decompression_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK || + pernix_inverse_scale_f32(scale, &inverse_scale) != PERNIX_STATUS_OK) { return 1; } - if (decompress_block(bit_width, compressed, scale, restored) != 0) { - return 1; + + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, inverse_scale, + compressed) != PERNIX_STATUS_OK) { + return 2; + } + if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, + true) != PERNIX_STATUS_OK) { + return 3; } - return 0; } ``` -### Fortran example (using bindings) +Complete examples are in `examples/cpp`, `examples/c`, and `examples/fortran`. + +## Fortran + +Fortran bindings are optional. Enable them with: + +```bash +cmake -S . -B build -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON +``` + +The modules in `bindings/fortran/src` bind directly to the C ABI names and currently expose f32 and f64 compression and +decompression entry points. The bindings do not yet install Fortran module files as a packaged Fortran SDK; they are +meant +for in-tree builds and examples. + +## Building + +Release build: + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --config Release -j"$(nproc)" +``` + +Common options: + +* `-DBUILD_SHARED_LIBS=OFF`: build a static library instead of the default shared library +* `-DPERNIX_ENABLE_TESTS=ON`: build tests +* `-DPERNIX_ENABLE_EXAMPLES=ON`: build examples +* `-DPERNIX_ENABLE_INSTALL=ON`: install library, headers, CMake package files, and pkg-config metadata +* `-DPERNIX_ENABLE_INSTALL_CONSUMER_TESTS=ON`: add a CTest that installs PERNIX and builds C/C++ consumers +* `-DPERNIX_ENABLE_FORTRAN_BINDINGS=ON`: build Fortran bindings and Fortran example/test +* `-DPERNIX_ENABLE_X86_AVX2=OFF`, `-DPERNIX_ENABLE_X86_BMI2=OFF`, + `-DPERNIX_ENABLE_X86_AVX512VBMI=OFF`: disable specific x86 backends +* `-DPERNIX_ENABLE_FALLBACK_STDPAR=OFF|AUTO|ON`: control the stdpar fallback backend. TBB is optional and enables + parallel standard execution policies when found +* `-DPERNIX_ENABLE_FALLBACK_SIMD=OFF|AUTO|ON`: control the experimental C++26 + `std::simd` fallback backend. `AUTO` enables it when both the compiler and standard + library support `std::simd`; `ON` makes missing support a configuration error -```fortran -program pernix_example - use iso_c_binding, only : c_int8_t, c_float, c_loc - use pernix_compression - use pernix_decompression - implicit none +Install: - integer(c_int8_t), parameter :: bit_width = 16_c_int8_t - real(c_float), parameter :: scale = 1.5_c_float - real(c_float), target :: input_data(32), output_data(32) - integer(c_int8_t), target :: compressed_data(64) - integer :: i +```bash +cmake --install build --prefix /path/to/prefix --config Release +``` + +CMake consumers can use: + +```cmake +find_package(pernix CONFIG REQUIRED) +target_link_libraries(my_target PRIVATE pernix::pernix) +``` - do i = 1, size(input_data) - input_data(i) = real(i, c_float) - end do +## Testing - call compress_block(bit_width, c_loc(input_data), scale, c_loc(compressed_data)) - call decompress_block(bit_width, c_loc(compressed_data), scale, c_loc(output_data)) -end program pernix_example +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DPERNIX_ENABLE_TESTS=ON -DPERNIX_ENABLE_EXAMPLES=ON +cmake --build build --config Release -j"$(nproc)" +ctest --test-dir build --output-on-failure ``` -For a complete Fortran binding setup, see `bindings/README.md` and `bindings/fortran/main.f90`. +Examples are built when `PERNIX_ENABLE_EXAMPLES=ON`: + +```bash +./build/examples/pernix_example_cpp +./build/examples/pernix_example_c +``` + +If Fortran bindings are enabled: + +```bash +./build/examples/pernix_example_fortran +``` + +## Limitations + +* Public documentation is centered on fixed 64-byte blocks; larger accepted block sizes are compatibility/internal + paths. +* Input values should be finite and within the intended quantization range for portable cross-backend behavior. The + scalar + fallback clamps NaN, infinity, and out-of-range scaled values, but that is not yet specified as a cross-backend + policy. +* Scale must be positive and finite. +* ARM64 compression backends are incomplete/stubbed. +* `PERNIX_BACKEND_FALLBACK_STDPAR` is not exported by the compiled library target. +* Fortran bindings are buildable in-tree but not yet packaged for installation. +* The packed format is intended to be stable for a given bit width and sign mode, but PERNIX is still pre-1.0. + +## Performance Notes -## Benchmarking -A benchmark framework for PERNIX can be found at https://github.com/pc2/pernix-benchmark. +Decompression is the performance-sensitive path. Prefer `PERNIX_BACKEND_AUTO` for normal use so PERNIX can select an +available optimized backend. Use `PERNIX_BACKEND_FALLBACK` when deterministic portable fallback behavior is more +important +than backend selection. +A separate benchmark framework exists at . diff --git a/bindings/README.md b/bindings/README.md index f3ae472..7866dad 100644 --- a/bindings/README.md +++ b/bindings/README.md @@ -1,14 +1,15 @@ ## Fortran bindings -Fortran bindings for the library are provided in the `bindings/fortran` directory. These bindings allow Fortran programs -to interface with the core functionality of the library. +Fortran bindings for the library are provided in the `bindings/fortran` directory. They expose the public C ABI symbols +such as `pernix_compress_block_f32`, `pernix_decompress_block_f32`, and the matching f64 variants. -### Building the Fortran Bindings +### Building the Fortran bindings -To build the Fortran bindings, you need to have a Fortran compiler installed on your system. The build process is -integrated into the main build system of the library. You can enable the Fortran bindings by passing the appropriate -flag to the build configuration command. For example, if you are using CMake, you can enable the Fortran bindings with: +To build the Fortran bindings, you need a Fortran compiler. Enable them from the main CMake build: ```bash -cmake -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON .. +cmake -S . -B build -DPERNIX_ENABLE_FORTRAN_BINDINGS=ON +cmake --build build --config Release -j"$(nproc)" ``` + +The `pernix_bindings_fortran_test` target performs one f32 64-byte block round trip through the Fortran modules. diff --git a/bindings/fortran/CMakeLists.txt b/bindings/fortran/CMakeLists.txt index ce9995b..c745cf8 100644 --- a/bindings/fortran/CMakeLists.txt +++ b/bindings/fortran/CMakeLists.txt @@ -1,19 +1,23 @@ -enable_language(Fortran) - file(GLOB_RECURSE SOURCE_FILES src/*.f90 ) add_library(pernix_bindings_fortran SHARED ${SOURCE_FILES}) -target_sources(pernix_bindings_fortran PRIVATE ${SOURCE_FILES}) +set_target_properties(pernix_bindings_fortran PROPERTIES + Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/mod" +) +target_include_directories(pernix_bindings_fortran PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/mod") target_link_libraries(pernix_bindings_fortran PUBLIC pernix) add_executable(pernix_bindings_fortran_test main.f90) target_link_libraries(pernix_bindings_fortran_test PRIVATE pernix_bindings_fortran pernix) -link_directories($ENV{LD_LIBRARY_PATH}) find_package(OpenMP) -if (OpenMP_CXX_FOUND) - target_link_libraries(pernix_bindings_fortran_test PUBLIC OpenMP::OpenMP_CXX) +if (OpenMP_Fortran_FOUND) + target_link_libraries(pernix_bindings_fortran_test PUBLIC OpenMP::OpenMP_Fortran) +endif () + +if (BUILD_TESTING) + add_test(NAME pernix_fortran_roundtrip COMMAND pernix_bindings_fortran_test) endif () diff --git a/bindings/fortran/main.f90 b/bindings/fortran/main.f90 index d1cac69..2f499e8 100644 --- a/bindings/fortran/main.f90 +++ b/bindings/fortran/main.f90 @@ -1,42 +1,36 @@ -! Hello World Fortran program -program main - use iso_c_binding, only : c_int8_t, c_int32_t, c_float, c_loc, c_ptr - use iso_fortran_env, only : real64, int64 - use pernix_decompression +program pernix_fortran_roundtrip + use iso_c_binding, only : c_bool, c_float, c_int, c_int8_t, c_int32_t, c_loc use pernix_compression - use omp_lib + use pernix_decompression implicit none - integer(c_int8_t) :: bit_width = 16_c_int8_t + integer(c_int8_t), parameter :: bit_width = 16_c_int8_t + integer(c_int32_t), parameter :: block_size = 64_c_int32_t + integer, parameter :: elements = (64 * 8) / 16 + real(c_float), target :: input(elements) + real(c_float), target :: restored(elements) + integer(c_int8_t), target :: compressed(block_size) + real(c_float) :: bmax real(c_float) :: scale - integer(c_int8_t), target :: input_data(512) - real(c_float), target :: output_data(512) - integer :: i, j, gb, iter - real(real64) :: t0, t1, diff + integer(c_int) :: status + integer :: i - ! Initialize example data - scale = 1.5_c_float - output_data = 0.0_c_float + do i = 1, elements + input(i) = real(i - 17, c_float) * 0.125_c_float + end do - iter = 50000000_int64 - t0 = omp_get_wtime() - do i = 1_int64, iter - do j = 1, size(input_data) - input_data(j) = mod(i + j, 256_int64) - end do + bmax = maxval(abs(input)) + scale = bmax / real((2 ** (int(bit_width) - 1)) - 1, c_float) - ! call the C-binding Fortran subroutine using C pointers - call decompress_block(bit_width, c_loc(input_data), scale, c_loc(output_data)) - end do - t1 = omp_get_wtime() - diff = t1 - t0 - gb = iter * (512_int64 + 1024_int64) / (1024_int64 * 1024_int64 * 1024_int64) + status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(input), & + 1.0_c_float / scale, c_loc(compressed)) + if (status /= PERNIX_STATUS_OK) stop 1 - print *, "Time taken for ", iter, " decompressions: ", diff, " seconds" - print *, "Throughput: ", gb / diff, " GB/s" + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(compressed), & + scale, c_loc(restored), .true._c_bool) + if (status /= PERNIX_STATUS_OK) stop 2 - ! Print the output data - ! do i = 1, size(output_data) - ! print *, "output_data(", i, ") = ", output_data(i) - ! end do -end program main \ No newline at end of file + do i = 1, elements + if (abs(restored(i) - input(i)) > scale) stop 3 + end do +end program pernix_fortran_roundtrip diff --git a/bindings/fortran/src/compression.f90 b/bindings/fortran/src/compression.f90 index 8261364..d294ee1 100644 --- a/bindings/fortran/src/compression.f90 +++ b/bindings/fortran/src/compression.f90 @@ -1,104 +1,60 @@ module pernix_compression - use iso_c_binding, only : c_int8_t, c_int32_t, c_float, c_ptr + use iso_c_binding, only : c_bool, c_double, c_float, c_int, c_int8_t, c_int32_t, c_ptr implicit none - interface - -#if (__SSE3__ && __SSE4_1__ && __SSE4_2__) -#if (__AVX__ && __AVX2__ && __FMA__ && __BMI__ && __BMI2__) - - subroutine mm256_compress_block_bmi2(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm256_compress_block_bmi2") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine mm256_compress_block_avx2(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm256_compress_block_avx2") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - + integer(c_int), parameter :: PERNIX_BACKEND_AUTO = 0_c_int + integer(c_int), parameter :: PERNIX_BACKEND_FALLBACK = 1_c_int + integer(c_int), parameter :: PERNIX_STATUS_OK = 0_c_int - subroutine mm256_compress_blocks_bmi2(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm256_compress_blocks_bmi2") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - - subroutine mm256_compress_blocks_avx2(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm256_compress_blocks_avx2") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - -#if (__AVX512BW__ && __AVX512CD__ && __AVX512DQ__ && __AVX512F__ && __AVX512VL__ && __AVX512VBMI__) - - subroutine mm512_compress_block_avx512vbmi(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm512_compress_block_avx512vbmi") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine mm512_compress_blocks_avx512vbmi(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm512_compress_blocks_avx512vbmi") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - -#endif -#endif -#endif - - subroutine compress_block_fallback(bit_width, input_ptr, scale, output_ptr) bind(C, name = "compress_block_fallback") - import :: c_int8_t, c_float, c_ptr + interface + function pernix_compress_block_f32(backend, bit_width, block_size, input_ptr, scale, output_ptr) & + bind(C, name = "pernix_compress_block_f32") result(status) + import :: c_float, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr real(c_float), value :: scale type(c_ptr), value :: output_ptr - end subroutine + integer(c_int) :: status + end function - subroutine compress_blocks_fallback(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "compress_blocks_fallback") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_compress_blocks_f32(backend, bit_width, block_size, input_ptr, scale, output_ptr, blocks) & + bind(C, name = "pernix_compress_blocks_f32") result(status) + import :: c_float, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr real(c_float), value :: scale type(c_ptr), value :: output_ptr integer(c_int32_t), value :: blocks - end subroutine + integer(c_int) :: status + end function - subroutine compress_block(bit_width, input_ptr, scale, output_ptr) bind(C, name = "compress_block") - import :: c_int8_t, c_float, c_ptr + function pernix_compress_block_f64(backend, bit_width, block_size, input_ptr, scale, output_ptr) & + bind(C, name = "pernix_compress_block_f64") result(status) + import :: c_double, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr - real(c_float), value :: scale + real(c_double), value :: scale type(c_ptr), value :: output_ptr - end subroutine + integer(c_int) :: status + end function - subroutine compress_blocks(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "compress_blocks") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_compress_blocks_f64(backend, bit_width, block_size, input_ptr, scale, output_ptr, blocks) & + bind(C, name = "pernix_compress_blocks_f64") result(status) + import :: c_double, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr - real(c_float), value :: scale + real(c_double), value :: scale type(c_ptr), value :: output_ptr integer(c_int32_t), value :: blocks - end subroutine - + integer(c_int) :: status + end function end interface - -end module pernix_compression \ No newline at end of file +end module pernix_compression diff --git a/bindings/fortran/src/decompression.f90 b/bindings/fortran/src/decompression.f90 index fb34e75..c2347fc 100644 --- a/bindings/fortran/src/decompression.f90 +++ b/bindings/fortran/src/decompression.f90 @@ -1,105 +1,60 @@ module pernix_decompression - use iso_c_binding, only : c_int8_t, c_int32_t, c_float, c_ptr + use iso_c_binding, only : c_bool, c_double, c_float, c_int, c_int8_t, c_int32_t, c_ptr implicit none interface - -#if (__SSE3__ && __SSE4_1__ && __SSE4_2__) -#if (__AVX__ && __AVX2__ && __FMA__ && __BMI__ && __BMI2__) - - subroutine mm256_decompress_block_bmi2(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm256_decompress_block_bmi2") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine mm256_decompress_block_avx2(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm256_decompress_block_avx2") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - - subroutine mm256_decompress_blocks_bmi2(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm256_decompress_blocks_bmi2") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_decompress_block_f32(backend, bit_width, block_size, input_ptr, scale, output_ptr, sign_values) & + bind(C, name = "pernix_decompress_block_f32") result(status) + import :: c_bool, c_float, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr real(c_float), value :: scale type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine + logical(c_bool), value :: sign_values + integer(c_int) :: status + end function - subroutine mm256_decompress_blocks_avx2(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm256_decompress_blocks_avx2") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_decompress_blocks_f32(backend, bit_width, block_size, input_ptr, scale, output_ptr, blocks, & + sign_values) bind(C, name = "pernix_decompress_blocks_f32") result(status) + import :: c_bool, c_float, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr real(c_float), value :: scale type(c_ptr), value :: output_ptr integer(c_int32_t), value :: blocks - end subroutine + logical(c_bool), value :: sign_values + integer(c_int) :: status + end function -#if (__AVX512BW__ && __AVX512CD__ && __AVX512DQ__ && __AVX512F__ && __AVX512VL__ && __AVX512VBMI__) - - subroutine mm512_decompress_block_avx512vbmi(bit_width, input_ptr, scale, output_ptr) bind(C, name = "mm512_decompress_block_avx512vbmi") - import :: c_int8_t, c_float, c_ptr + function pernix_decompress_block_f64(backend, bit_width, block_size, input_ptr, scale, output_ptr, sign_values) & + bind(C, name = "pernix_decompress_block_f64") result(status) + import :: c_bool, c_double, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr - real(c_float), value :: scale + real(c_double), value :: scale type(c_ptr), value :: output_ptr - end subroutine + logical(c_bool), value :: sign_values + integer(c_int) :: status + end function - subroutine mm512_decompress_blocks_avx512vbmi(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "mm512_decompress_blocks_avx512vbmi") - import :: c_int8_t, c_int32_t, c_float, c_ptr + function pernix_decompress_blocks_f64(backend, bit_width, block_size, input_ptr, scale, output_ptr, blocks, & + sign_values) bind(C, name = "pernix_decompress_blocks_f64") result(status) + import :: c_bool, c_double, c_int, c_int8_t, c_int32_t, c_ptr + integer(c_int), value :: backend integer(c_int8_t), value :: bit_width + integer(c_int32_t), value :: block_size type(c_ptr), value :: input_ptr - real(c_float), value :: scale + real(c_double), value :: scale type(c_ptr), value :: output_ptr integer(c_int32_t), value :: blocks - end subroutine - -#endif -#endif -#endif - - subroutine decompress_block_fallback(bit_width, input_ptr, scale, output_ptr) bind(C, name = "decompress_block_fallback") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine decompress_blocks_fallback(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "decompress_blocks_fallback") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - - subroutine decompress_block(bit_width, input_ptr, scale, output_ptr) bind(C, name = "decompress_block") - import :: c_int8_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - end subroutine - - subroutine decompress_blocks(bit_width, input_ptr, scale, output_ptr, blocks) bind(C, name = "decompress_blocks") - import :: c_int8_t, c_int32_t, c_float, c_ptr - integer(c_int8_t), value :: bit_width - type(c_ptr), value :: input_ptr - real(c_float), value :: scale - type(c_ptr), value :: output_ptr - integer(c_int32_t), value :: blocks - end subroutine - - + logical(c_bool), value :: sign_values + integer(c_int) :: status + end function end interface - -end module pernix_decompression \ No newline at end of file +end module pernix_decompression diff --git a/cmake/pernix.pc.in b/cmake/pernix.pc.in index 87fdb27..00f60cd 100644 --- a/cmake/pernix.pc.in +++ b/cmake/pernix.pc.in @@ -1,11 +1,12 @@ -prefix=@CMAKE_INSTALL_PREFIX@ +prefix=${pcfiledir}/../.. exec_prefix=${prefix} -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ +libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ name: Pernix description: A high-performance C++ library for de/compression of floating-point data version: @NORMALIZED_VERSION@ +Requires: @PERNIX_PC_REQUIRES@ Libs: -L${libdir} -lpernix -Cflags: -I${includedir} \ No newline at end of file +Cflags: -I${includedir} diff --git a/cmake/pernixConfig.cmake.in b/cmake/pernixConfig.cmake.in new file mode 100644 index 0000000..6819581 --- /dev/null +++ b/cmake/pernixConfig.cmake.in @@ -0,0 +1,24 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) + +if (@PERNIX_USE_SIMDE@) + find_package(simde CONFIG QUIET) + if (NOT TARGET simde::simde AND @PERNIX_BUNDLE_SIMDE_FOR_INSTALL@) + add_library(simde::simde INTERFACE IMPORTED) + target_include_directories(simde::simde INTERFACE "${PACKAGE_PREFIX_DIR}/include") + endif () + if (NOT TARGET simde::simde) + set(pernix_FOUND FALSE) + set(pernix_NOT_FOUND_MESSAGE "pernix was built with SIMDe support, but simde::simde was not found. Install SIMDe's CMake package or use a Pernix package built with bundled SIMDe headers.") + return() + endif () +endif () + +if (@PERNIX_BUILD_FALLBACK_STDPAR@) + find_dependency(TBB) +endif () + +include("${CMAKE_CURRENT_LIST_DIR}/pernixTargets.cmake") + +check_required_components(pernix) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 0000000..5021306 --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,10 @@ +add_executable(pernix_example_cpp cpp/roundtrip.cpp) +target_link_libraries(pernix_example_cpp PRIVATE pernix) + +add_executable(pernix_example_c c/roundtrip.c) +target_link_libraries(pernix_example_c PRIVATE pernix) + +if (PERNIX_ENABLE_FORTRAN_BINDINGS) + add_executable(pernix_example_fortran fortran/roundtrip.f90) + target_link_libraries(pernix_example_fortran PRIVATE pernix_bindings_fortran pernix) +endif () diff --git a/examples/c/roundtrip.c b/examples/c/roundtrip.c new file mode 100644 index 0000000..e1660b5 --- /dev/null +++ b/examples/c/roundtrip.c @@ -0,0 +1,44 @@ +#include + +int main(void) { + enum { bit_width = 16, block_size = 64, elements = (block_size * 8) / bit_width }; + float input[elements]; + float restored[elements]; + u8 compressed[block_size]; + float bmax = 0.0f; + + if (!pernix_is_valid_bit_width(bit_width) || pernix_compressed_block_size() != block_size || + pernix_elements_per_block(bit_width) != elements) { + return 1; + } + + for (int i = 0; i < elements; ++i) { + input[i] = ((float)i - 16.0f) * 0.125f; + const float magnitude = input[i] < 0.0f ? -input[i] : input[i]; + bmax = bmax < magnitude ? magnitude : bmax; + } + + float scale = 0.0f; + float inverse_scale = 0.0f; + if (pernix_decompression_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK || + pernix_inverse_scale_f32(scale, &inverse_scale) != PERNIX_STATUS_OK) { + return 2; + } + + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { + return 3; + } + if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, true) != + PERNIX_STATUS_OK) { + return 4; + } + + for (int i = 0; i < elements; ++i) { + const float diff = restored[i] - input[i]; + const float error = diff < 0.0f ? -diff : diff; + if (error > scale) { + return 5; + } + } + return 0; +} diff --git a/examples/cpp/roundtrip.cpp b/examples/cpp/roundtrip.cpp new file mode 100644 index 0000000..5365937 --- /dev/null +++ b/examples/cpp/roundtrip.cpp @@ -0,0 +1,44 @@ +#include +#include +#include +#include +#include + +int main() { + constexpr u8 bit_width = 16; + constexpr u32 block_size = pernix::compressed_block_size(); + constexpr usize elements = pernix::elements_per_block(bit_width); + + std::array input{}; + for (usize i = 0; i < input.size(); ++i) { + input[i] = std::sin(static_cast(i) * 0.25f); + } + + float bmax = 0.0f; + for (const float value : input) { + bmax = std::max(bmax, std::abs(value)); + } + float scale = 0.0f; + float inverse_scale = 0.0f; + if (pernix::decompression_scale_from_bmax(bmax, bit_width, scale) != PERNIX_STATUS_OK || + pernix::inverse_scale(scale, inverse_scale) != PERNIX_STATUS_OK) { + return 1; + } + + std::array compressed{}; + std::array restored{}; + + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, inverse_scale, compressed) != PERNIX_STATUS_OK) { + return 1; + } + if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, scale, restored) != PERNIX_STATUS_OK) { + return 2; + } + + for (usize i = 0; i < input.size(); ++i) { + if (std::abs(restored[i] - input[i]) > scale) { + return 3; + } + } + return 0; +} diff --git a/examples/fortran/roundtrip.f90 b/examples/fortran/roundtrip.f90 new file mode 100644 index 0000000..033ec1c --- /dev/null +++ b/examples/fortran/roundtrip.f90 @@ -0,0 +1,38 @@ +program pernix_example_fortran + use iso_c_binding, only : c_bool, c_float, c_int, c_int8_t, c_int32_t, c_loc + use pernix_compression + use pernix_decompression + implicit none + + integer(c_int8_t), parameter :: bit_width = 16_c_int8_t + integer(c_int32_t), parameter :: block_size = 64_c_int32_t + integer, parameter :: elements = (64 * 8) / 16 + real(c_float), target :: input(elements) + real(c_float), target :: restored(elements) + integer(c_int8_t), target :: compressed(block_size) + real(c_float) :: bmax + real(c_float) :: scale + real(c_float) :: inverse_scale + real(c_float) :: error + integer(c_int) :: status + integer :: i + + do i = 1, elements + input(i) = sin(real(i, c_float) * 0.25_c_float) + end do + + bmax = maxval(abs(input)) + scale = bmax / real((2 ** (int(bit_width) - 1)) - 1, c_float) + inverse_scale = 1.0_c_float / scale + + status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(input), & + inverse_scale, c_loc(compressed)) + if (status /= PERNIX_STATUS_OK) stop 1 + + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, c_loc(compressed), & + scale, c_loc(restored), .true._c_bool) + if (status /= PERNIX_STATUS_OK) stop 2 + + error = maxval(abs(restored - input)) + if (error > scale) stop 3 +end program pernix_example_fortran diff --git a/external/simde b/external/simde deleted file mode 160000 index 1747b24..0000000 --- a/external/simde +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1747b2482589fe894d49989159421da08c2a8bcd diff --git a/include/pernix/arm64/neon/common.h b/include/pernix/arm64/neon/common.h new file mode 100644 index 0000000..63d15bb --- /dev/null +++ b/include/pernix/arm64/neon/common.h @@ -0,0 +1,214 @@ +#ifndef PERNIX_ARM64_NEON_COMMON_H +#define PERNIX_ARM64_NEON_COMMON_H + +#include + +#include + +namespace pernix::arm64::neon::internal { +struct float64x2x8_t { + float64x2_t val[8]; +}; + +static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements) { + const u32 tail_bits = remaining_elements * bit_width; + const u32 tail_bytes = (tail_bits + 7u) / 8u; + return tail_bytes; +} + +__always_inline int32x4x4_t neon_convert_int8x16_int32x4x4(int8x16_t input) { + const int16x8_t s16_lo = vmovl_s8(vget_low_s8(input)); + const int16x8_t s16_hi = vmovl_s8(vget_high_s8(input)); + + return { + { + vmovl_s16(vget_low_s16(s16_lo)), + vmovl_s16(vget_high_s16(s16_lo)), + vmovl_s16(vget_low_s16(s16_hi)), + vmovl_s16(vget_high_s16(s16_hi)), + } + }; +} + +__always_inline int32x4x2_t neon_convert_int16x8_int32x4x2(int16x8_t input) { + return { + { + vmovl_s16(vget_low_s16(input)), + vmovl_s16(vget_high_s16(input)), + } + }; +} + +__always_inline float32x4x4_t neon_dequantize_epi32(const int32x4x4_t& input, float32x4_t scale) { + return { + { + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[2]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[3]), scale), + } + }; +} + +__always_inline float32x4x2_t neon_dequantize_epi32(const int32x4x2_t& input, float32x4_t scale) { + return { + { + vmulq_f32(vcvtq_f32_s32(input.val[0]), scale), + vmulq_f32(vcvtq_f32_s32(input.val[1]), scale), + } + }; +} + +__always_inline float32x4_t neon_dequantize_epi32(int32x4_t input, float32x4_t scale) { + return vmulq_f32(vcvtq_f32_s32(input), scale); +} + +__always_inline float64x2_t neon_dequantize_epi32_f64(int32x2_t input, float64x2_t scale) { + return vmulq_f64(vcvtq_f64_s64(vmovl_s32(input)), scale); +} + +__always_inline float64x2x2_t neon_dequantize_epi32_f64(int32x4_t input, float64x2_t scale) { + return { + { + neon_dequantize_epi32_f64(vget_low_s32(input), scale), + neon_dequantize_epi32_f64(vget_high_s32(input), scale), + } + }; +} + +__always_inline float64x2x4_t neon_dequantize_epi32_f64(const int32x4x2_t& input, float64x2_t scale) { + const float64x2x2_t dequantized_low = neon_dequantize_epi32_f64(input.val[0], scale); + const float64x2x2_t dequantized_high = neon_dequantize_epi32_f64(input.val[1], scale); + + return { + { + dequantized_low.val[0], + dequantized_low.val[1], + dequantized_high.val[0], + dequantized_high.val[1], + } + }; +} + +__always_inline float64x2x8_t neon_dequantize_epi32_f64(const int32x4x4_t& input, float64x2_t scale) { + const float64x2x2_t dequantized0 = neon_dequantize_epi32_f64(input.val[0], scale); + const float64x2x2_t dequantized1 = neon_dequantize_epi32_f64(input.val[1], scale); + const float64x2x2_t dequantized2 = neon_dequantize_epi32_f64(input.val[2], scale); + const float64x2x2_t dequantized3 = neon_dequantize_epi32_f64(input.val[3], scale); + + return { + { + dequantized0.val[0], + dequantized0.val[1], + dequantized1.val[0], + dequantized1.val[1], + dequantized2.val[0], + dequantized2.val[1], + dequantized3.val[0], + dequantized3.val[1], + } + }; +} + +__always_inline uint8x16_t neon_load_tail_elements_int8(const u8* input, const u32 tail_bytes_count) { + u8 buffer[16] = {0}; + std::memcpy(buffer, input, tail_bytes_count); + return vld1q_u8(buffer); +} + +__always_inline uint16x8_t neon_load_tail_elements_int16(const u8* input, const u32 tail_bytes_count) { + u16 buffer[8] = {0}; + std::memcpy(buffer, input, tail_bytes_count); + return vld1q_u16(buffer); +} + +__always_inline uint32x4_t neon_load_tail_elements_int32(const u8* input, const u32 tail_bytes_count) { + u32 buffer[4] = {0}; + std::memcpy(buffer, input, tail_bytes_count); + return vld1q_u32(buffer); +} + +__always_inline float32x4_t neon_load_tail_elements_f32(const u8* input, const u32 tail_elements) { + float32_t buffer[4] = {0.0f}; + std::memcpy(buffer, input, tail_elements * sizeof(float32_t)); + return vld1q_f32(buffer); +} + +__always_inline float64x2_t neon_load_tail_elements_f64(const u8* input, const u32 tail_elements) { + float64_t buffer[2] = {0.0}; + std::memcpy(buffer, input, tail_elements * sizeof(float64_t)); + return vld1q_f64(buffer); +} + +__always_inline void neon_store_tail_elements_int8(u8* output, const uint8x16x4_t& data, const u32 tail_elements) { + u8 buffer[16 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_u8(buffer + i * 16, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(u8)); +} + +__always_inline void neon_store_tail_elements_int16(u16* output, const uint16x8x4_t& data, const u32 tail_elements) { + u16 buffer[8 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_u16(buffer + i * 8, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(u16)); +} + +__always_inline void neon_store_tail_elements_int32(u32* output, const uint32x4x4_t& data, const u32 tail_elements) { + u32 buffer[4 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_u32(buffer + i * 4, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(u32)); +} + +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x4_t& data, const u32 tail_elements) { + float32_t buffer[16 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_f32(buffer + i * 4, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); +} + +__always_inline void neon_store_tail_elements_f32(float32_t* output, const float32x4x2_t& data, const u32 tail_elements) { + float32_t buffer[8 * 2]; + for (u32 i = 0; i < 2; ++i) { + vst1q_f32(buffer + i * 4, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); +} + +__always_inline void neon_store_tail_elements_f32(float32_t* output, float32x4_t data, const u32 tail_elements) { + float32_t buffer[4]; + vst1q_f32(buffer, data); + std::memcpy(output, buffer, tail_elements * sizeof(float32_t)); +} + +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x4_t& data, const u32 tail_elements) { + float64_t buffer[2 * 4]; + for (u32 i = 0; i < 4; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); +} + +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x2_t& data, const u32 tail_elements) { + float64_t buffer[2 * 2]; + for (u32 i = 0; i < 2; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); +} + +__always_inline void neon_store_tail_elements_f64(float64_t* output, const float64x2x8_t& data, const u32 tail_elements) { + float64_t buffer[2 * 8]; + for (u32 i = 0; i < 8; ++i) { + vst1q_f64(buffer + i * 2, data.val[i]); + } + std::memcpy(output, buffer, tail_elements * sizeof(float64_t)); +} +} // namespace pernix::arm64::neon::internal + +#endif // PERNIX_ARM64_NEON_COMMON_H diff --git a/include/pernix/arm64/neon/compression.h b/include/pernix/arm64/neon/compression.h new file mode 100644 index 0000000..63adcb2 --- /dev/null +++ b/include/pernix/arm64/neon/compression.h @@ -0,0 +1,111 @@ +#ifndef PERNIX_ARM64_NEON_COMPRESSION_H +#define PERNIX_ARM64_NEON_COMPRESSION_H + +#include +#include + +#include +#include + +namespace pernix::arm64::neon { +namespace internal { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_1to8(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_9to16(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_17to24(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_1to8(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_9to16(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block_17to24(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { + static_assert(true, "Not yet implemented"); + return -1; +} +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_compress_block_17to24(input, scale, output); + } + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_compress_block(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_compress_block_1to8(input, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_compress_block_9to16(input, scale, output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_compress_block_17to24(input, scale, output); + } + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_compress_blocks(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output, const u32 blocks) { + const u8* block_input = input; + f32* block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + neon_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_compress_blocks(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output, const u32 blocks) { + const u8* block_input = input; + f64* block_output = output; + + for (u32 block = 0; block < blocks; ++block) { + neon_compress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + return 0; +} +} // namespace pernix::arm64::neon + +#endif // PERNIX_ARM64_NEON_COMPRESSION_H diff --git a/include/pernix/arm64/neon/decompression.h b/include/pernix/arm64/neon/decompression.h new file mode 100644 index 0000000..f688f6a --- /dev/null +++ b/include/pernix/arm64/neon/decompression.h @@ -0,0 +1,344 @@ +#ifndef PERNIX_ARM64_NEON_DECOMPRESSION_H +#define PERNIX_ARM64_NEON_DECOMPRESSION_H + +#include +#include +#include + +#include +#include + +namespace pernix::arm64::neon { +namespace internal { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_16 = elements_per_block / 16; + constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16; + + const float32x4_t scale_v = vdupq_n_f32(scale); + + for (uint32_t i = 0; i < iterations_16; ++i) { + const uint8x16_t source = vld1q_u8(input); + const int8x16_t unpacked = b128::neon_unpack_epi8_1to8(source); + + const int32x4x4_t converted = neon_convert_int8x16_int32x4x4(unpacked); + const float32x4x4_t dequantized = neon_dequantize_epi32(converted, scale_v); + + for (uint32_t j = 0; j < 4; ++j) { + vst1q_f32(output, dequantized.val[j]); + output += 4; + } + + input += 2 * BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const uint8x16_t tail_source = neon_load_tail_elements_int8(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8(tail_source); + + const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x4(tail_unpacked); + const float32x4x4_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); + + neon_store_tail_elements_f32(output, tail_dequantized, remaining_elements); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_8 = elements_per_block / 8; + constexpr uint32_t remaining_elements = elements_per_block - iterations_8 * 8; + + const float32x4_t scale_v = vdupq_n_f32(scale); + + for (uint32_t i = 0; i < iterations_8; ++i) { + const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); + const int16x8_t unpacked = b128::neon_unpack_epi16_9to16(source); + + const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); + const float32x4x2_t dequantized = neon_dequantize_epi32(converted, scale_v); + + for (uint32_t j = 0; j < 2; ++j) { + vst1q_f32(output, dequantized.val[j]); + output += 4; + } + + input += BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16(tail_source); + + const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); + const float32x4x2_t tail_dequantized = neon_dequantize_epi32(tail_converted, scale_v); + + neon_store_tail_elements_f32(output, tail_dequantized, remaining_elements); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_4 = elements_per_block / 4; + constexpr uint32_t remaining_elements = elements_per_block - iterations_4 * 4; + + const float32x4_t scale_v = vdupq_n_f32(scale); + + for (uint32_t i = 0; i < iterations_4; ++i) { + const uint32_t group_bit_start = i * 4u * BIT_WIDTH; + const uint8_t* group_input = input + group_bit_start / 8u; + const uint32x4_t source = vld1q_u32(reinterpret_cast(group_input)); + + int32x4_t unpacked; + if constexpr (BIT_WIDTH % 2 == 0) { + unpacked = b128::neon_unpack_epi32_17to24(source); + } else { + if (i % 2 == 0) { + unpacked = b128::neon_unpack_epi32_17to24(source); + } else { + unpacked = b128::neon_unpack_epi32_17to24(source); + } + } + + const float32x4_t dequantized = neon_dequantize_epi32(unpacked, scale_v); + + vst1q_f32(output, dequantized); + + output += 4; + } + + if constexpr (remaining_elements > 0) { + constexpr uint32_t tail_bit_start = iterations_4 * 4u * BIT_WIDTH; + constexpr uint32_t tail_bit_offset = tail_bit_start % 8u; + const uint8_t* tail_input = input + tail_bit_start / 8u; + + constexpr uint32_t tail_bytes_count = (tail_bit_offset + remaining_elements * BIT_WIDTH + 7u) / 8u; + const uint32x4_t tail_source = neon_load_tail_elements_int32(tail_input, tail_bytes_count); + + int32x4_t tail_unpacked; + if constexpr (tail_bit_offset == 0) { + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); + } else { + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); + } + + const float32x4_t tail_dequantized = neon_dequantize_epi32(tail_unpacked, scale_v); + + neon_store_tail_elements_f32(output, tail_dequantized, remaining_elements); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_16 = elements_per_block / 16; + constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16; + + const float64x2_t scale_v = vdupq_n_f64(scale); + + for (uint32_t i = 0; i < iterations_16; ++i) { + const uint8x16_t source = vld1q_u8(input); + const int8x16_t unpacked = b128::neon_unpack_epi8_1to8(source); + + const int32x4x4_t converted = neon_convert_int8x16_int32x4x4(unpacked); + const float64x2x8_t dequantized = neon_dequantize_epi32_f64(converted, scale_v); + + for (uint32_t j = 0; j < 8; ++j) { + vst1q_f64(output, dequantized.val[j]); + output += 2; + } + + input += 2 * BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const uint8x16_t tail_source = neon_load_tail_elements_int8(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int8x16_t tail_unpacked = b128::neon_unpack_epi8_1to8(tail_source); + + const int32x4x4_t tail_converted = neon_convert_int8x16_int32x4x4(tail_unpacked); + const float64x2x8_t tail_dequantized = neon_dequantize_epi32_f64(tail_converted, scale_v); + + neon_store_tail_elements_f64(output, tail_dequantized, remaining_elements); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_8 = elements_per_block / 8; + constexpr uint32_t remaining_elements = elements_per_block - iterations_8 * 8; + + const float64x2_t scale_v = vdupq_n_f64(scale); + + for (uint32_t i = 0; i < iterations_8; ++i) { + const uint16x8_t source = vld1q_u16(reinterpret_cast(input)); + const int16x8_t unpacked = b128::neon_unpack_epi16_9to16(source); + + const int32x4x2_t converted = neon_convert_int16x8_int32x4x2(unpacked); + const float64x2x4_t dequantized = neon_dequantize_epi32_f64(converted, scale_v); + + for (uint32_t j = 0; j < 4; ++j) { + vst1q_f64(output, dequantized.val[j]); + output += 2; + } + + input += BIT_WIDTH; + } + + if constexpr (remaining_elements > 0) { + const uint16x8_t tail_source = neon_load_tail_elements_int16(input, tail_bytes(BIT_WIDTH, remaining_elements)); + const int16x8_t tail_unpacked = b128::neon_unpack_epi16_9to16(tail_source); + + const int32x4x2_t tail_converted = neon_convert_int16x8_int32x4x2(tail_unpacked); + const float64x2x4_t tail_dequantized = neon_dequantize_epi32_f64(tail_converted, scale_v); + + neon_store_tail_elements_f64(output, tail_dequantized, remaining_elements); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + constexpr uint32_t iterations_4 = elements_per_block / 4; + constexpr uint32_t remaining_elements = elements_per_block - iterations_4 * 4; + + const float64x2_t scale_v = vdupq_n_f64(scale); + + for (uint32_t i = 0; i < iterations_4; ++i) { + const uint32_t group_bit_start = i * 4u * BIT_WIDTH; + const uint8_t* group_input = input + group_bit_start / 8u; + const uint32x4_t source = vld1q_u32(reinterpret_cast(group_input)); + + int32x4_t unpacked; + if constexpr (BIT_WIDTH % 2 == 0) { + unpacked = b128::neon_unpack_epi32_17to24(source); + } else { + if (i % 2 == 0) { + unpacked = b128::neon_unpack_epi32_17to24(source); + } else { + unpacked = b128::neon_unpack_epi32_17to24(source); + } + } + + const float64x2x2_t dequantized = neon_dequantize_epi32_f64(unpacked, scale_v); + + for (uint32_t j = 0; j < 2; ++j) { + vst1q_f64(output, dequantized.val[j]); + output += 2; + } + } + + if constexpr (remaining_elements > 0) { + constexpr uint32_t tail_bit_start = iterations_4 * 4u * BIT_WIDTH; + constexpr uint32_t tail_bit_offset = tail_bit_start % 8u; + const uint8_t* tail_input = input + tail_bit_start / 8u; + + constexpr uint32_t tail_bytes_count = (tail_bit_offset + remaining_elements * BIT_WIDTH + 7u) / 8u; + const uint32x4_t tail_source = neon_load_tail_elements_int32(tail_input, tail_bytes_count); + + int32x4_t tail_unpacked; + if constexpr (tail_bit_offset == 0) { + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); + } else { + tail_unpacked = b128::neon_unpack_epi32_17to24(tail_source); + } + + const float64x2x2_t tail_dequantized = neon_dequantize_epi32_f64(tail_unpacked, scale_v); + + neon_store_tail_elements_f64(output, tail_dequantized, remaining_elements); + } + + return 0; +} +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block(const void* __restrict__ input, const float_t scale, void* __restrict__ output) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_decompress_block_1to8(typed_input, scale, typed_output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_decompress_block_9to16(typed_input, scale, typed_output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_decompress_block_17to24(typed_input, scale, typed_output); + } + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int neon_decompress_block(const void* __restrict__ input, const double_t scale, void* __restrict__ output) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::neon_decompress_block_1to8(typed_input, scale, typed_output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::neon_decompress_block_9to16(typed_input, scale, typed_output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::neon_decompress_block_17to24(typed_input, scale, typed_output); + } + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_decompress_blocks(const void* __restrict__ input, const float_t scale, void* __restrict__ output, const uint32_t blocks) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + const uint8_t* block_input = typed_input; + float_t* block_output = typed_output; + + for (uint32_t block = 0; block < blocks; ++block) { + neon_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int neon_decompress_blocks(const void* __restrict__ input, const double_t scale, void* __restrict__ output, const uint32_t blocks) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + const uint8_t* block_input = typed_input; + double_t* block_output = typed_output; + + for (uint32_t block = 0; block < blocks; ++block) { + neon_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; +} +} // namespace pernix::arm64::neon + +#endif // PERNIX_ARM64_NEON_DECOMPRESSION_H diff --git a/include/pernix/arm64/neon/packing.h b/include/pernix/arm64/neon/packing.h new file mode 100644 index 0000000..a972dcf --- /dev/null +++ b/include/pernix/arm64/neon/packing.h @@ -0,0 +1,8 @@ +#ifndef PERNIX_ARM64_NEON_PACKING_H +#define PERNIX_ARM64_NEON_PACKING_H + +#include + +namespace pernix::arm64::neon::internal {} // namespace pernix::arm64::neon::internal + +#endif // PERNIX_ARM64_NEON_PACKING_H diff --git a/include/pernix/arm64/neon/tables.h b/include/pernix/arm64/neon/tables.h new file mode 100644 index 0000000..32da264 --- /dev/null +++ b/include/pernix/arm64/neon/tables.h @@ -0,0 +1,207 @@ +#ifndef PERNIX_ARM64_NEON_TABLES_H +#define PERNIX_ARM64_NEON_TABLES_H + +#include + +#include +#include +#include + +namespace pernix::arm64::neon::internal { +namespace detail { +inline constexpr usize neon_vector_width = 128; +inline constexpr u8 inactive_lane = 0xff; + +template +constexpr bool table_indices_are_valid(const std::array& table) { + return std::ranges::all_of(table, [](const u8 index) { return index == inactive_lane || index < Elements; }); +} + +template +constexpr std::array make_primary_permute() { + static_assert(LANE_BITS % 8 == 0); + + constexpr usize lane_bytes = LANE_BITS / 8; + static_assert(ELEMENTS % lane_bytes == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (usize entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const usize bit_start = entry * BIT_WIDTH; + const usize first_byte = bit_start / 8; + const usize base = entry * lane_bytes; + + for (usize lane_byte = 0; lane_byte < lane_bytes; ++lane_byte) { + table[base + lane_byte] = static_cast(first_byte + lane_byte); + } + } + + return table; +} + +template +constexpr std::array make_spill_permute() { + static_assert(LANE_BITS % 8 == 0); + + constexpr usize lane_bytes = LANE_BITS / 8; + static_assert(ELEMENTS % lane_bytes == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (usize entry = 0; entry < ELEMENTS / lane_bytes; ++entry) { + const usize bit_start = entry * BIT_WIDTH; + const usize first_byte = bit_start / 8; + const usize bit_offset = bit_start % 8; + const usize base = entry * lane_bytes; + + if (bit_offset + BIT_WIDTH > LANE_BITS) { + table[base] = static_cast(first_byte + lane_bytes); + } + } + + return table; +} + +template +constexpr std::array make_shift_right() { + std::array table{}; + table.fill(0); + + for (usize entry = 0; entry < ELEMENTS; ++entry) { + const usize bit_start = entry * BIT_WIDTH; + const usize bit_offset = bit_start % 8u; + + table[entry] = -static_cast(bit_offset); + } + + return table; +} + +template +constexpr std::array make_shift_left_for_spill() { + std::array table{}; + table.fill(0); + + for (usize entry = 0; entry < ELEMENTS; ++entry) { + const usize bit_start = entry * BIT_WIDTH; + const usize bit_offset = bit_start % 8u; + const bool spills = bit_offset + BIT_WIDTH > LANE_BITS; + + table[entry] = spills ? static_cast(LANE_BITS - bit_offset) : 0; + } + + return table; +} + +template +constexpr std::array make_contiguous_permute_32() { + static_assert(ELEMENTS % 4 == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (usize entry = 0; entry < ELEMENTS / 4; ++entry) { + const usize bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + const usize bit_end = bit_start + BIT_WIDTH - 1; + const usize first_byte = bit_start / 8; + const usize last_byte = bit_end / 8; + const usize base = entry * 4; + + for (usize byte = first_byte; byte <= last_byte; ++byte) { + table[base + (byte - first_byte)] = static_cast(byte); + } + } + + return table; +} + +template +constexpr std::array make_shift_right_32() { + std::array table{}; + table.fill(0); + + for (usize entry = 0; entry < ELEMENTS; ++entry) { + const usize bit_start = START_BIT_OFFSET + entry * BIT_WIDTH; + + table[entry] = -static_cast(bit_start % 8u); + } + + return table; +} +} // namespace detail + +template +struct table_unpacking; + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && VECTOR_WIDTH == detail::neon_vector_width) +struct table_unpacking { +private: + static constexpr usize PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr usize SHIFT_ELEMENTS = VECTOR_WIDTH / 8; + +public: + static constexpr u8 bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute1 = detail::make_primary_permute(); + alignas(64) static constexpr std::array permute2 = detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); + alignas(64) static constexpr std::array shift2 = + detail::make_shift_left_for_spill(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 16); + static_assert(detail::table_indices_are_valid(permute1)); + static_assert(detail::table_indices_are_valid(permute2)); +}; + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && VECTOR_WIDTH == detail::neon_vector_width) +struct table_unpacking { +private: + static constexpr usize PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr usize SHIFT_ELEMENTS = VECTOR_WIDTH / 16; + +public: + static constexpr u8 bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute1 = + detail::make_primary_permute(); + alignas(64) static constexpr std::array permute2 = detail::make_spill_permute(); + alignas(64) static constexpr std::array shift1 = detail::make_shift_right(); + alignas(64) static constexpr std::array shift2 = + detail::make_shift_left_for_spill(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 8); + static_assert(detail::table_indices_are_valid(permute1)); + static_assert(detail::table_indices_are_valid(permute2)); +}; + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && VECTOR_WIDTH == detail::neon_vector_width && START_BIT_OFFSET < 8) +struct table_unpacking { +private: + static constexpr usize PERMUTE_ELEMENTS = VECTOR_WIDTH / 8; + static constexpr usize SHIFT_ELEMENTS = VECTOR_WIDTH / 32; + +public: + static constexpr u8 bit_width = BIT_WIDTH; + + alignas(64) static constexpr std::array permute = + detail::make_contiguous_permute_32(); + alignas(64) static constexpr std::array shift = + detail::make_shift_right_32(); + + static_assert(PERMUTE_ELEMENTS == 16); + static_assert(SHIFT_ELEMENTS == 4); + static_assert(detail::table_indices_are_valid(permute)); +}; + +template +struct table_packing; +} // namespace pernix::arm64::neon::internal + +#endif // PERNIX_ARM64_NEON_TABLES_H diff --git a/include/pernix/arm64/neon/unpacking.h b/include/pernix/arm64/neon/unpacking.h new file mode 100644 index 0000000..93b8994 --- /dev/null +++ b/include/pernix/arm64/neon/unpacking.h @@ -0,0 +1,100 @@ +#ifndef PERNIX_ARM64_NEON_UNPACKING_H +#define PERNIX_ARM64_NEON_UNPACKING_H + +#include +#include + +using namespace pernix::arm64::neon::internal; + +namespace pernix::arm64::neon::internal::b128 { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline int8x16_t neon_unpack_epi8_1to8(uint8x16_t input) { + if constexpr (BIT_WIDTH == 8) { + return vreinterpretq_s8_u8(input); + } else if constexpr (BIT_WIDTH == 1) { + using tables = table_unpacking; + + const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); + const uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); + + return vreinterpretq_s8_u8(vandq_u8(shifted, vdupq_n_u8(1))); + } else { + using tables = table_unpacking; + + const uint8x16_t permuted_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute1.data())); + + uint8x16_t shifted = vshlq_u8(permuted_u8, vld1q_s8(tables::shift1.data())); + + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + const uint8x16_t permuted2_u8 = vqtbl1q_u8(input, vld1q_u8(tables::permute2.data())); + + shifted = vorrq_u8(shifted, vshlq_u8(permuted2_u8, vld1q_s8(tables::shift2.data()))); + } + + constexpr int shift = 8 - BIT_WIDTH; + shifted = vshlq_n_u8(shifted, shift); + + if constexpr (SIGN_VALUES) { + return vshlq_s8(vreinterpretq_s8_u8(shifted), vdupq_n_s8(-shift)); + } else { + return vreinterpretq_s8_u8(vshlq_u8(shifted, vdupq_n_s8(-shift))); + } + } +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline int16x8_t neon_unpack_epi16_9to16(uint16x8_t input) { + if constexpr (BIT_WIDTH == 16) { + return vreinterpretq_s16_u16(input); + } else { + using tables = table_unpacking; + + const uint8x16_t input_u8 = vreinterpretq_u8_u16(input); + + const uint8x16_t permuted1_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute1.data())); + + uint16x8_t shifted = vshlq_u16(vreinterpretq_u16_u8(permuted1_u8), vld1q_s16(tables::shift1.data())); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const uint8x16_t permuted2_u8 = vqtbl1q_u8(input_u8, vld1q_u8(tables::permute2.data())); + + const uint16x8_t shifted2 = vshlq_u16(vreinterpretq_u16_u8(permuted2_u8), vld1q_s16(tables::shift2.data())); + + shifted = vorrq_u16(shifted, shifted2); + } + + constexpr int shift = 16 - BIT_WIDTH; + shifted = vshlq_n_u16(shifted, shift); + + if constexpr (SIGN_VALUES) { + return vshlq_s16(vreinterpretq_s16_u16(shifted), vdupq_n_s16(-shift)); + } else { + return vreinterpretq_s16_u16(vshlq_u16(shifted, vdupq_n_s16(-shift))); + } + } +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline int32x4_t neon_unpack_epi32_17to24(uint32x4_t input) { + using tables = table_unpacking; + + const uint8x16_t input_8 = vreinterpretq_u8_u32(input); + + const uint8x16_t permuted_u8 = vqtbl1q_u8(input_8, vld1q_u8(tables::permute.data())); + + const uint32x4_t value = vshlq_u32(vreinterpretq_u32_u8(permuted_u8), vld1q_s32(tables::shift.data())); + + if constexpr (SIGN_VALUES) { + constexpr int sign_shift = 32 - BIT_WIDTH; + return vshrq_n_s32(vreinterpretq_s32_u32(vshlq_n_u32(value, sign_shift)), sign_shift); + } else { + constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1u; + return vreinterpretq_s32_u32(vandq_u32(value, vdupq_n_u32(mask))); + } +} +} // namespace pernix::arm64::neon::internal::b128 + +#endif // PERNIX_ARM64_NEON_UNPACKING_H diff --git a/include/pernix/arm64/sve2/compression.h b/include/pernix/arm64/sve2/compression.h new file mode 100644 index 0000000..8ddc0b9 --- /dev/null +++ b/include/pernix/arm64/sve2/compression.h @@ -0,0 +1,44 @@ +#ifndef PERNIX_ARM64_SVE2_COMPRESSION_H +#define PERNIX_ARM64_SVE2_COMPRESSION_H + +#include + +#include +#include + +namespace pernix { +namespace internal { +template +inline constexpr bool sve2_compression_unimplemented_v = false; +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_block(const f32*, f32, u8*) { + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_block(const f64*, f64, u8*) { + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_blocks(const f32*, f32, u8*, u32) { + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + return -1; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_compress_blocks(const f64*, f64, u8*, u32) { + static_assert(internal::sve2_compression_unimplemented_v, "ARM64 SVE2 compression is not implemented yet"); + return -1; +} +} // namespace pernix + +#endif // PERNIX_ARM64_SVE2_COMPRESSION_H diff --git a/include/pernix/arm64/sve2/decompression.h b/include/pernix/arm64/sve2/decompression.h new file mode 100644 index 0000000..3effd31 --- /dev/null +++ b/include/pernix/arm64/sve2/decompression.h @@ -0,0 +1,427 @@ +#ifndef PERNIX_ARM64_SVE2_DECOMPRESSION_H +#define PERNIX_ARM64_SVE2_DECOMPRESSION_H + +#include +#include +#include + +#include +#include +#include +#include + +namespace pernix::arm64::sve2 { +namespace internal { +template +[[nodiscard]] __always_inline constexpr uint32_t packed_bytes(const uint32_t elements) { + return (elements * BIT_WIDTH + 7) / 8; +} + +[[nodiscard]] __always_inline svuint8_t sve2_load_packed_bytes(const uint8_t* __restrict__ input, const uint32_t bytes) { + const svbool_t pg = svwhilelt_b8(uint64_t{0}, static_cast(bytes)); + return svld1_u8(pg, input); +} + +template +__always_inline void sve2_store_dequantized_i8_f32(svint8_t values, const svfloat32_t scale_v, float_t* __restrict__ output, + const uint32_t count) { + alignas(64) std::vector temp(svcntb()); + + svst1_s8(svptrue_b8(), temp.data(), values); + + uint32_t offset = 0; + while (offset < count) { + const svbool_t pg = svwhilelt_b32(static_cast(offset), static_cast(count)); + + svfloat32_t dequantized; + + if constexpr (SIGN_VALUES) { + const svint32_t widened = svld1sb_s32(pg, temp.data() + offset); + dequantized = svmul_f32_x(pg, svcvt_f32_s32_x(pg, widened), scale_v); + } else { + const svuint32_t widened = svld1ub_u32(pg, reinterpret_cast(temp.data() + offset)); + dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, widened), scale_v); + } + + svst1_f32(pg, output + offset, dequantized); + + offset += static_cast(svcntw()); + } +} + +template +__always_inline void sve2_store_dequantized_i8_f64(svint8_t values, const double_t scale, double_t* __restrict__ output, + const uint32_t count) { + std::vector temp(svcntb()); + + svst1_s8(svptrue_b8(), temp.data(), values); + + for (uint32_t i = 0; i < count; ++i) { + if constexpr (SIGN_VALUES) { + output[i] = static_cast(temp[i]) * scale; + } else { + output[i] = static_cast(static_cast(temp[i])) * scale; + } + } +} + +template +__always_inline void sve2_store_dequantized_i16_f32(svint16_t values, const svfloat32_t scale_v, float_t* __restrict__ output, + const uint32_t count) { + alignas(64) std::vector temp(svcnth()); + + svst1_s16(svptrue_b16(), temp.data(), values); + + uint32_t offset = 0; + while (offset < count) { + const svbool_t pg = svwhilelt_b32(static_cast(offset), static_cast(count)); + + svfloat32_t dequantized; + if constexpr (SIGN_VALUES) { + const svint32_t widened = svld1sh_s32(pg, temp.data() + offset); + dequantized = svmul_f32_x(pg, svcvt_f32_s32_x(pg, widened), scale_v); + } else { + const svuint32_t widened = svld1uh_u32(pg, reinterpret_cast(temp.data() + offset)); + dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, widened), scale_v); + } + + svst1_f32(pg, output + offset, dequantized); + + offset += static_cast(svcntw()); + } +} + +template +__always_inline void sve2_store_dequantized_i16_f64(svint16_t values, const double_t scale, double_t* __restrict__ output, + const uint32_t count) { + std::vector temp(svcnth()); + + svst1_s16(svptrue_b16(), temp.data(), values); + + for (uint32_t i = 0; i < count; ++i) { + if constexpr (SIGN_VALUES) { + output[i] = static_cast(temp[i]) * scale; + } else { + output[i] = static_cast(static_cast(temp[i])) * scale; + } + } +} + +template +__always_inline void sve2_store_dequantized_i32_f32(svint32_t values, const svfloat32_t scale_v, float_t* __restrict__ output, + const uint32_t count) { + const svbool_t pg = svwhilelt_b32(uint64_t{0}, static_cast(count)); + + svfloat32_t dequantized; + if constexpr (SIGN_VALUES) { + dequantized = svmul_f32_x(pg, svcvt_f32_s32_x(pg, values), scale_v); + } else { + dequantized = svmul_f32_x(pg, svcvt_f32_u32_x(pg, svreinterpret_u32_s32(values)), scale_v); + } + + svst1_f32(pg, output, dequantized); +} + +template +__always_inline void sve2_store_dequantized_i32_f64(svint32_t values, const double_t scale, double_t* __restrict__ output, + const uint32_t count) { + std::vector temp(svcntw()); + + svst1_s32(svptrue_b32(), temp.data(), values); + + for (uint32_t i = 0; i < count; ++i) { + if constexpr (SIGN_VALUES) { + output[i] = static_cast(temp[i]) * scale; + } else { + output[i] = static_cast(static_cast(temp[i])) * scale; + } + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcntb()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const svfloat32_t scale_v = svdup_n_f32(scale); + + const table_unpacking table; + const svuint8_t permute = table.permute(); + const svuint8_t shift = table.shift(); + svuint8_t spill_permute = svdup_n_u8(0); + svuint8_t spill_shift = svdup_n_u8(0); + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + spill_permute = table.spill_permute(); + spill_shift = table.spill_shift(); + } + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint32_t bytes = packed_bytes(count); + const uint8_t* chunk_input = input + input_bit_offset / 8; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svint8_t unpacked = sve2_unpack_epi8_1to8(source, permute, shift, spill_permute, spill_shift); + + sve2_store_dequantized_i8_f32(unpacked, scale_v, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcnth()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const svfloat32_t scale_v = svdup_n_f32(scale); + + const table_unpacking table; + const svuint8_t permute = table.permute(); + const svuint16_t shift = table.shift(); + svuint8_t spill_permute = svdup_n_u8(0); + svuint16_t spill_shift = svdup_n_u16(0); + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + spill_permute = table.spill_permute(); + spill_shift = table.spill_shift(); + } + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint32_t bytes = packed_bytes(count); + const uint8_t* chunk_input = input + input_bit_offset / 8; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svint16_t unpacked = + sve2_unpack_epi16_9to16(svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); + + sve2_store_dequantized_i16_f32(unpacked, scale_v, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcntw()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const svfloat32_t scale_v = svdup_n_f32(scale); + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint8_t* chunk_input = input + input_bit_offset / 8; + const uint32_t bit_offset = input_bit_offset % 8; + const uint32_t bytes = (bit_offset + count * BIT_WIDTH + 7u) / 8u; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + svint32_t unpacked; + if (bit_offset == 0) { + unpacked = sve2_unpack_epi32_17to24(source); + } else { + unpacked = sve2_unpack_epi32_17to24(source); + } + + sve2_store_dequantized_i32_f32(unpacked, scale_v, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_1to8(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcntb()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const table_unpacking table; + const svuint8_t permute = table.permute(); + const svuint8_t shift = table.shift(); + svuint8_t spill_permute = svdup_n_u8(0); + svuint8_t spill_shift = svdup_n_u8(0); + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + spill_permute = table.spill_permute(); + spill_shift = table.spill_shift(); + } + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint32_t bytes = packed_bytes(count); + const uint8_t* chunk_input = input + input_bit_offset / 8; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svint8_t unpacked = sve2_unpack_epi8_1to8(source, permute, shift, spill_permute, spill_shift); + + sve2_store_dequantized_i8_f64(unpacked, scale, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_9to16(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcnth()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + const table_unpacking table; + const svuint8_t permute = table.permute(); + const svuint16_t shift = table.shift(); + svuint8_t spill_permute = svdup_n_u8(0); + svuint16_t spill_shift = svdup_n_u16(0); + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + spill_permute = table.spill_permute(); + spill_shift = table.spill_shift(); + } + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint32_t bytes = packed_bytes(count); + const uint8_t* chunk_input = input + input_bit_offset / 8; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + const svint16_t unpacked = + sve2_unpack_epi16_9to16(svreinterpret_u16_u8(source), permute, shift, spill_permute, spill_shift); + + sve2_store_dequantized_i16_f64(unpacked, scale, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +__always_inline int sve2_decompress_block_17to24(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { + constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto lanes = static_cast(svcntw()); + uint32_t input_bit_offset = 0; + uint32_t processed_elements = 0; + + while (processed_elements < elements_per_block) { + const uint32_t count = std::min(elements_per_block - processed_elements, lanes); + + const uint8_t* chunk_input = input + input_bit_offset / 8; + const uint32_t bit_offset = input_bit_offset % 8; + const uint32_t bytes = (bit_offset + count * BIT_WIDTH + 7u) / 8u; + + const svuint8_t source = sve2_load_packed_bytes(chunk_input, bytes); + svint32_t unpacked; + if (bit_offset == 0) { + unpacked = sve2_unpack_epi32_17to24(source); + } else { + unpacked = sve2_unpack_epi32_17to24(source); + } + + sve2_store_dequantized_i32_f64(unpacked, scale, output + processed_elements, count); + + processed_elements += count; + input_bit_offset += count * BIT_WIDTH; + } + + return 0; +} +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_block(const void* __restrict__ input, const float_t scale, void* __restrict__ output) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::sve2_decompress_block_1to8(typed_input, scale, typed_output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::sve2_decompress_block_9to16(typed_input, scale, typed_output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::sve2_decompress_block_17to24(typed_input, scale, typed_output); + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_block(const void* __restrict__ input, const double_t scale, void* __restrict__ output) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + return internal::sve2_decompress_block_1to8(typed_input, scale, typed_output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + return internal::sve2_decompress_block_9to16(typed_input, scale, typed_output); + } else if constexpr (BIT_WIDTH >= 17 && BIT_WIDTH <= 24) { + return internal::sve2_decompress_block_17to24(typed_input, scale, typed_output); + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_blocks(const void* __restrict__ input, const float_t scale, void* __restrict__ output, const uint32_t blocks) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + const uint8_t* block_input = typed_input; + float_t* block_output = typed_output; + + for (uint32_t block = 0; block < blocks; ++block) { + sve2_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int sve2_decompress_blocks(const void* __restrict__ input, const double_t scale, void* __restrict__ output, const uint32_t blocks) { + const auto* typed_input = static_cast(input); + auto* typed_output = static_cast(output); + const uint8_t* block_input = typed_input; + double_t* block_output = typed_output; + + for (uint32_t block = 0; block < blocks; ++block) { + sve2_decompress_block(block_input, scale, block_output); + block_input += BLOCK_SIZE; + block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; + } + + return 0; +} +} // namespace pernix::arm64::sve2 + +#endif // PERNIX_ARM64_SVE2_DECOMPRESSION_H diff --git a/include/pernix/arm64/sve2/packing.h b/include/pernix/arm64/sve2/packing.h new file mode 100644 index 0000000..802c612 --- /dev/null +++ b/include/pernix/arm64/sve2/packing.h @@ -0,0 +1,11 @@ +#ifndef PERNIX_ARM64_SVE2_PACKING_H +#define PERNIX_ARM64_SVE2_PACKING_H + +#include + +namespace pernix::arm64::sve2::internal { +template +inline constexpr bool packing_unimplemented_v = false; +} // namespace pernix::arm64::sve2::internal + +#endif // PERNIX_ARM64_SVE2_PACKING_H diff --git a/include/pernix/arm64/sve2/tables.h b/include/pernix/arm64/sve2/tables.h new file mode 100644 index 0000000..e6b0a79 --- /dev/null +++ b/include/pernix/arm64/sve2/tables.h @@ -0,0 +1,115 @@ +#ifndef PERNIX_ARM64_SVE2_TABLES_H +#define PERNIX_ARM64_SVE2_TABLES_H + +#include + +namespace pernix::arm64::sve2::internal { +template +struct table_unpacking { + static constexpr u8 bit_width = BIT_WIDTH; + + static svbool_t pg_b8() { return svptrue_b8(); } + + static svbool_t pg_b16() { return svptrue_b16(); } + + static svbool_t pg_b32() { return svptrue_b32(); } +}; + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +struct table_unpacking { + static constexpr u8 bit_width = BIT_WIDTH; + + static svuint8_t permute() { + const svbool_t pg = svptrue_b8(); + return svlsr_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 3); + } + + static svuint8_t spill_permute() { + const svbool_t pg = svptrue_b8(); + return svadd_n_u8_x(pg, permute(), 1); + } + + static svuint8_t shift() { + const svbool_t pg = svptrue_b8(); + return svand_n_u8_x(pg, svindex_u8(0, BIT_WIDTH), 7); + } + + static svuint8_t spill_shift() { + const svbool_t pg = svptrue_b8(); + return svsub_u8_x(pg, svdup_n_u8(8), shift()); + } +}; + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +struct table_unpacking { + static constexpr u8 bit_width = BIT_WIDTH; + + static svuint8_t permute() { + const svbool_t pg = svptrue_b8(); + const svuint8_t lane = svindex_u8(0, 1); + const svuint8_t elem = svlsr_n_u8_x(pg, lane, 1); + const svuint8_t byte = svand_n_u8_x(pg, lane, 1); + + svuint8_t first; + if constexpr (BIT_WIDTH == 16) { + first = svlsl_n_u8_x(pg, elem, 1); + } else { + constexpr u8 extra_bits = BIT_WIDTH - 8u; + const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); + const svuint8_t low = svlsr_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), 3); + first = svadd_u8_x(pg, elem, svadd_u8_x(pg, high, low)); + } + + return svadd_u8_x(pg, first, byte); + } + + static svuint8_t spill_permute() { + const svbool_t pg = svptrue_b8(); + return svadd_n_u8_x(pg, permute(), 2); + } + + static svuint16_t shift() { + const svbool_t pg = svptrue_b16(); + return svand_n_u16_x(pg, svmul_n_u16_x(pg, svindex_u16(0, 1), BIT_WIDTH), 7); + } + + static svuint16_t spill_shift() { + const svbool_t pg = svptrue_b16(); + const svuint16_t bit_shift = shift(); + const svuint16_t spill = svsub_u16_x(pg, svdup_n_u16(16), bit_shift); + return svsel_u16(svcmpgt_n_u16(pg, bit_shift, 16u - BIT_WIDTH), spill, svdup_n_u16(16)); + } +}; + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && START_BIT_OFFSET < 8) +struct table_unpacking { + static constexpr u8 bit_width = BIT_WIDTH; + + static svuint8_t permute() { + const svbool_t pg = svptrue_b8(); + const svuint8_t lane = svindex_u8(0, 1); + const svuint8_t elem = svlsr_n_u8_x(pg, lane, 2); + const svuint8_t byte = svand_n_u8_x(pg, lane, 3); + + svuint8_t first = svmul_n_u8_x(pg, elem, BIT_WIDTH / 8u); + if constexpr (BIT_WIDTH % 8u != 0) { + constexpr u8 extra_bits = BIT_WIDTH % 8u; + const svuint8_t high = svmul_n_u8_x(pg, svlsr_n_u8_x(pg, elem, 3), extra_bits); + const svuint8_t low_bits = svadd_n_u8_x(pg, svmul_n_u8_x(pg, svand_n_u8_x(pg, elem, 7), extra_bits), START_BIT_OFFSET); + first = svadd_u8_x(pg, first, svadd_u8_x(pg, high, svlsr_n_u8_x(pg, low_bits, 3))); + } + + return svadd_u8_x(pg, first, byte); + } + + static svuint32_t shift() { + const svbool_t pg = svptrue_b32(); + return svand_n_u32_x(pg, svadd_n_u32_x(pg, svmul_n_u32_x(pg, svindex_u32(0, 1), BIT_WIDTH), START_BIT_OFFSET), 7); + } +}; +} // namespace pernix::arm64::sve2::internal + +#endif // PERNIX_ARM64_SVE2_TABLES_H diff --git a/include/pernix/arm64/sve2/unpacking.h b/include/pernix/arm64/sve2/unpacking.h new file mode 100644 index 0000000..824a2b6 --- /dev/null +++ b/include/pernix/arm64/sve2/unpacking.h @@ -0,0 +1,92 @@ +#ifndef PERNIX_ARM64_SVE2_UNPACKING_H +#define PERNIX_ARM64_SVE2_UNPACKING_H + +#include + +#include "tables.h" + +namespace pernix::arm64::sve2::internal { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) +__always_inline svint8_t sve2_unpack_epi8_1to8(const svuint8_t input, const svuint8_t permute, const svuint8_t shift, + const svuint8_t spill_permute, const svuint8_t spill_shift) { + if constexpr (BIT_WIDTH == 8) { + return svreinterpret_s8(input); + } else { + const svbool_t pg = svptrue_b8(); + + const svuint8_t permuted = svtbl_u8(input, permute); + svuint8_t unpacked = svlsr_u8_x(pg, permuted, shift); + + if constexpr (BIT_WIDTH == 3 || BIT_WIDTH == 5 || BIT_WIDTH == 6 || BIT_WIDTH == 7) { + const svuint8_t spill_permuted_values = svtbl_u8(input, spill_permute); + const svuint8_t spill_shifted = svlsl_u8_x(pg, spill_permuted_values, spill_shift); + unpacked = svorr_u8_x(pg, unpacked, spill_shifted); + } + + if constexpr (BIT_WIDTH == 1) { + unpacked = svand_n_u8_x(pg, unpacked, 1); + return svreinterpret_s8(unpacked); + } else { + constexpr int sign_shift = 8 - BIT_WIDTH; + + unpacked = svlsl_n_u8_x(pg, unpacked, sign_shift); + + if constexpr (SIGN_VALUES) { + return svasr_n_s8_x(pg, svreinterpret_s8_u8(unpacked), sign_shift); + } else { + return svreinterpret_s8_u8(svlsr_n_u8_x(pg, unpacked, sign_shift)); + } + } + } +} + +template + requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) +__always_inline svint16_t sve2_unpack_epi16_9to16(const svuint16_t input, const svuint8_t permute, const svuint16_t shift, + const svuint8_t spill_permute, const svuint16_t spill_shift) { + if constexpr (BIT_WIDTH == 16) { + return svreinterpret_s16(input); + } else { + const svbool_t pg = svptrue_b16(); + + const svuint8_t permuted = svtbl_u8(svreinterpret_u8_u16(input), permute); + svuint16_t shifted = svlsr_u16_x(pg, svreinterpret_u16_u8(permuted), shift); + + if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { + const svuint8_t spill_permuted_values = svtbl_u8(svreinterpret_u8_u16(input), spill_permute); + const svuint16_t spill_shifted = svlsl_u16_x(pg, svreinterpret_u16_u8(spill_permuted_values), spill_shift); + shifted = svorr_u16_x(pg, shifted, spill_shifted); + } + + constexpr int sign_shift = 16 - BIT_WIDTH; + shifted = svlsl_n_u16_x(pg, shifted, sign_shift); + + if constexpr (SIGN_VALUES) { + return svasr_n_s16_x(pg, svreinterpret_s16_u16(shifted), sign_shift); + } else { + return svreinterpret_s16_u16(svlsr_n_u16_x(pg, shifted, sign_shift)); + } + } +} + +template + requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) +__always_inline svint32_t sve2_unpack_epi32_17to24(const svuint8_t input) { + using table = table_unpacking; + + const svbool_t pg = svptrue_b32(); + const svuint8_t permuted = svtbl_u8(input, table::permute()); + const svuint32_t unpacked = svlsr_u32_x(pg, svreinterpret_u32_u8(permuted), table::shift()); + + if constexpr (SIGN_VALUES) { + constexpr int sign_shift = 32 - BIT_WIDTH; + return svasr_n_s32_x(pg, svreinterpret_s32_u32(svlsl_n_u32_x(pg, unpacked, sign_shift)), sign_shift); + } else { + constexpr u32 mask = (u32{1} << BIT_WIDTH) - 1u; + return svreinterpret_s32_u32(svand_n_u32_x(pg, unpacked, mask)); + } +} +} // namespace pernix::arm64::sve2::internal + +#endif // PERNIX_ARM64_SVE2_UNPACKING_H diff --git a/include/pernix/backend.hpp b/include/pernix/backend.hpp new file mode 100644 index 0000000..9c3892d --- /dev/null +++ b/include/pernix/backend.hpp @@ -0,0 +1,21 @@ +#ifndef PERNIX_BACKEND_HPP +#define PERNIX_BACKEND_HPP + +#include + +namespace pernix { +enum class Backend : u8 { + Auto = PERNIX_BACKEND_AUTO, + Fallback = PERNIX_BACKEND_FALLBACK, + FallbackScalar = PERNIX_BACKEND_FALLBACK_SCALAR, + X86Avx2 = PERNIX_BACKEND_X86_AVX2, + X86Bmi2 = PERNIX_BACKEND_X86_BMI2, + X86Avx512Vbmi = PERNIX_BACKEND_X86_AVX512_VBMI, + Arm64Neon = PERNIX_BACKEND_ARM64_NEON, + Arm64Sve = PERNIX_BACKEND_ARM64_SVE, + FallbackStdpar = PERNIX_BACKEND_FALLBACK_STDPAR, + FallbackSimd = PERNIX_BACKEND_FALLBACK_SIMD, +}; +} // namespace pernix + +#endif // PERNIX_BACKEND_HPP diff --git a/include/pernix/compat.h b/include/pernix/compat.h new file mode 100644 index 0000000..fc79a03 --- /dev/null +++ b/include/pernix/compat.h @@ -0,0 +1,51 @@ +#ifndef PERNIX_COMPAT_H +#define PERNIX_COMPAT_H + +#ifdef __cplusplus +#include +#include +#include +#else +#include +#include +#include +#endif + +#ifndef __always_inline +#if defined(__GNUC__) || defined(__clang__) +#define __always_inline inline __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define __always_inline __forceinline +#else +#define __always_inline inline +#endif +#endif + +#if defined(_WIN32) && defined(PERNIX_SHARED) +#if defined(PERNIX_BUILD_LIB) +#define PERNIX_API __declspec(dllexport) +#else +#define PERNIX_API __declspec(dllimport) +#endif +#else +#define PERNIX_API +#endif + +// Convenient type declarations +using u8 = std::uint8_t; +using u16 = std::uint16_t; +using u32 = std::uint32_t; +using u64 = std::uint64_t; +using uptr = std::uintptr_t; + +using i8 = std::int8_t; +using i16 = std::int16_t; +using i32 = std::int32_t; +using i64 = std::int64_t; + +using f32 = std::float_t; +using f64 = std::double_t; + +using usize = std::size_t; + +#endif // PERNIX_COMPAT_H diff --git a/include/pernix/detail/api.hpp b/include/pernix/detail/api.hpp new file mode 100644 index 0000000..41046c6 --- /dev/null +++ b/include/pernix/detail/api.hpp @@ -0,0 +1,217 @@ +#ifndef PERNIX_DETAIL_API_HPP +#define PERNIX_DETAIL_API_HPP + +#include +#include + +#include +#include +#include +#include + +namespace pernix::detail { +constexpr u8 min_bit_width = 1; +constexpr u8 max_bit_width = 24; +constexpr u32 fixed_block_size = 64; + +constexpr bool is_valid_bit_width(const u8 bit_width) { + return bit_width >= min_bit_width && bit_width <= max_bit_width; +} + +constexpr bool is_valid_block_size(const u32 block_size) { + return block_size == fixed_block_size || block_size == 128 || block_size == 256 || block_size == 512 || block_size == 1024; +} + +constexpr u32 elements_per_block(const u8 bit_width, const u32 block_size = fixed_block_size) { + return is_valid_bit_width(bit_width) && is_valid_block_size(block_size) ? (block_size * 8U) / bit_width : 0; +} + +template +constexpr ScaleType quantization_levels(const u8 bit_width) { + return bit_width == 1 ? ScaleType{1} : static_cast((1U << (bit_width - 1U)) - 1U); +} + +template +inline pernix_status scale_from_bmax(const ScaleType bmax, const u8 bit_width, ScaleType* scale) { + if (scale == nullptr || !is_valid_bit_width(bit_width) || !std::isfinite(bmax) || bmax < ScaleType{0}) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + if (bmax <= ScaleType{0}) { + *scale = std::numeric_limits::epsilon(); + } else { + *scale = bmax / quantization_levels(bit_width); + } + return PERNIX_STATUS_OK; +} + +template +inline bool is_valid_scale(const ScaleType scale) { + return std::isfinite(scale) && scale > ScaleType{0}; +} + +template +inline pernix_status inverse_scale(const ScaleType scale, ScaleType* inverse_scale_value) { + if (inverse_scale_value == nullptr || !is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + *inverse_scale_value = ScaleType{1} / scale; + return PERNIX_STATUS_OK; +} + +template +inline pernix_status compression_scale_from_bmax(const ScaleType bmax, const u8 bit_width, ScaleType* inverse_scale_value) { + ScaleType scale = ScaleType{0}; + const auto status = scale_from_bmax(bmax, bit_width, &scale); + if (status != PERNIX_STATUS_OK) { + return status; + } + return inverse_scale(scale, inverse_scale_value); +} + +inline const char* status_string(const pernix_status status) { + switch (status) { + case PERNIX_STATUS_OK: + return "PERNIX_STATUS_OK"; + case PERNIX_STATUS_INVALID_ARGUMENT: + return "PERNIX_STATUS_INVALID_ARGUMENT"; + case PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH: + return "PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH"; + case PERNIX_STATUS_UNSUPPORTED_BACKEND: + return "PERNIX_STATUS_UNSUPPORTED_BACKEND"; + case PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE: + return "PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE"; + case PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION: + return "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION"; + default: + return "PERNIX_STATUS_UNKNOWN"; + } +} + +inline pernix_status select_error_status(const std::string_view kernel_name) { + if (kernel_name == "invalid_backend") { + return PERNIX_STATUS_UNSUPPORTED_BACKEND; + } + if (kernel_name == "unsupported_implementation") { + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; + } + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; +} + +template +inline pernix_status compress_block(const Backend backend, const u8 bit_width, const u32 block_size, const void* input, + const ScaleType scale, void* output) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + if (!is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + const auto kernel = [&] { + if constexpr (std::is_same_v) { + return pernix::internal::select_compress_block_f32(backend, bit_width, block_size); + } else { + return pernix::internal::select_compress_block_f64(backend, bit_width, block_size); + } + }(); + if (!kernel) { + return select_error_status(kernel.name); + } + + return static_cast(kernel.func(input, scale, output)); +} + +template +inline pernix_status compress_blocks(const Backend backend, const u8 bit_width, const u32 block_size, const void* input, + const ScaleType scale, void* output, const u32 blocks) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (blocks == 0) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + if (!is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + const auto kernel = [&] { + if constexpr (std::is_same_v) { + return pernix::internal::select_compress_blocks_f32(backend, bit_width, block_size); + } else { + return pernix::internal::select_compress_blocks_f64(backend, bit_width, block_size); + } + }(); + if (!kernel) { + return select_error_status(kernel.name); + } + + return static_cast(kernel.func(input, scale, output, blocks)); +} + +template +inline pernix_status decompress_block(const Backend backend, const u8 bit_width, const u32 block_size, const void* input, + const ScaleType scale, void* output, const bool sign_values) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + if (!is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + const auto kernel = [&] { + if constexpr (std::is_same_v) { + return pernix::internal::select_decompress_block_f32(backend, bit_width, block_size, sign_values); + } else { + return pernix::internal::select_decompress_block_f64(backend, bit_width, block_size, sign_values); + } + }(); + if (!kernel) { + return select_error_status(kernel.name); + } + + return static_cast(kernel.func(input, scale, output)); +} + +template +inline pernix_status decompress_blocks(const Backend backend, const u8 bit_width, const u32 block_size, const void* input, + const ScaleType scale, void* output, const u32 blocks, const bool sign_values) { + if (input == nullptr || output == nullptr) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (blocks == 0) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + if (!is_valid_scale(scale)) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + const auto kernel = [&] { + if constexpr (std::is_same_v) { + return pernix::internal::select_decompress_blocks_f32(backend, bit_width, block_size, sign_values); + } else { + return pernix::internal::select_decompress_blocks_f64(backend, bit_width, block_size, sign_values); + } + }(); + if (!kernel) { + return select_error_status(kernel.name); + } + + return static_cast(kernel.func(input, scale, output, blocks)); +} +} // namespace pernix::detail + +#endif // PERNIX_DETAIL_API_HPP diff --git a/include/pernix/detail/bits.h b/include/pernix/detail/bits.h new file mode 100644 index 0000000..c9f87cf --- /dev/null +++ b/include/pernix/detail/bits.h @@ -0,0 +1,15 @@ +#ifndef PERNIX_DETAIL_BITS_H +#define PERNIX_DETAIL_BITS_H + +#include + +namespace pernix::detail { +template +constexpr u32 low_bit_mask() { + static_assert(BitWidth >= 1, "low_bit_mask requires at least one bit"); + static_assert(BitWidth < 32, "low_bit_mask requires fewer than 32 bits"); + return (u32{1} << BitWidth) - 1u; +} +} // namespace pernix::detail + +#endif // PERNIX_DETAIL_BITS_H diff --git a/include/pernix/detail/pack_tables.h b/include/pernix/detail/pack_tables.h new file mode 100644 index 0000000..e970828 --- /dev/null +++ b/include/pernix/detail/pack_tables.h @@ -0,0 +1,165 @@ +#ifndef PERNIX_PACK_TABLES_H +#define PERNIX_PACK_TABLES_H + +#include + +#include +#include +#include +#include +#include + +namespace pernix::detail { +namespace internal { +template +constexpr std::array make_primary_permute() { + constexpr usize lane_bits = sizeof(LaneType) * 8; + std::array table{}; + table.fill(static_cast(-1)); + + for (usize output_lane = 0; output_lane < Elements; ++output_lane) { + const usize output_start = output_lane * lane_bits; + const usize output_end = output_start + lane_bits; + usize contributor = 0; + + for (usize input_lane = 0; input_lane < Elements; ++input_lane) { + const usize input_start = input_lane * BitWidth; + const usize input_end = input_start + BitWidth; + + // Half-open intervals overlap when each starts before the other ends. + if (!(input_start < output_end && input_end > output_start)) { + continue; + } + + const auto shift = static_cast(input_start) - static_cast(output_start); + // Nonnegative shifts place input bits into this output lane and are primary contributors. + // Negative shifts cross the output boundary and are emitted by the spill logic instead. + if (shift >= 0 && contributor++ == Contributor) { + table[output_lane] = static_cast(input_lane); + break; + } + } + } + return table; +} + +template +constexpr std::array make_primary_shift() { + constexpr usize lane_bits = sizeof(LaneType) * 8; + const auto permute = make_primary_permute(); + std::array table{}; + table.fill(static_cast(lane_bits)); + + for (usize output_lane = 0; output_lane < Elements; ++output_lane) { + if (permute[output_lane] >= 0) { + table[output_lane] = static_cast(static_cast(permute[output_lane]) * BitWidth - output_lane * lane_bits); + } + } + return table; +} + +template +constexpr std::array make_spill_permute() { + constexpr usize lane_bits = sizeof(LaneType) * 8; + std::array table{}; + table.fill(static_cast(-1)); + + for (usize output_lane = 0; output_lane < Elements; ++output_lane) { + const usize output_start = output_lane * lane_bits; + for (usize input_lane = 0; input_lane < Elements; ++input_lane) { + const usize input_start = input_lane * BitWidth; + const usize input_end = input_start + BitWidth; + if (input_start < output_start && input_end > output_start) { + table[output_lane] = static_cast(input_lane); + break; + } + } + } + return table; +} + +template +constexpr std::array make_spill_shift() { + constexpr usize lane_bits = sizeof(LaneType) * 8; + const auto permute = make_spill_permute(); + std::array table{}; + table.fill(static_cast(lane_bits)); + + for (usize output_lane = 0; output_lane < Elements; ++output_lane) { + if (permute[output_lane] >= 0) { + table[output_lane] = static_cast(output_lane * lane_bits - static_cast(permute[output_lane]) * BitWidth); + } + } + return table; +} + +template +constexpr u64 make_active_mask(const std::array& permute) { + static_assert(Elements <= 64); + u64 mask = 0; + for (usize lane = 0; lane < Elements; ++lane) { + if (permute[lane] >= 0) { + mask |= u64{1} << lane; + } + } + return mask; +} + +template +constexpr std::array, sizeof...(Contributors)> make_primary_permutes(std::index_sequence) { + return {make_primary_permute()...}; +} + +template +constexpr std::array, sizeof...(Contributors)> make_primary_shifts(std::index_sequence) { + return {make_primary_shift()...}; +} + +template +constexpr std::array make_primary_masks(const std::array, Contributors>& permutes) { + std::array masks{}; + for (usize contributor = 0; contributor < Contributors; ++contributor) { + masks[contributor] = make_active_mask(permutes[contributor]); + } + return masks; +} +} // namespace internal + +// Unsupported lane types, bit widths, and vector sizes intentionally fail the focused assertions below. +template +struct pack_table { + static_assert(std::signed_integral, "pack_table LaneType must be a signed integral type"); + static_assert(sizeof(LaneType) == sizeof(u8) || sizeof(LaneType) == sizeof(u16) || sizeof(LaneType) == sizeof(u32), + "pack_table LaneType must use 8-, 16-, or 32-bit lanes"); + static_assert(VectorBytes > 0, "pack_table VectorBytes must be greater than zero"); + static_assert(VectorBytes % sizeof(LaneType) == 0, "pack_table VectorBytes must contain a whole number of lanes"); + + static constexpr usize lane_bits = sizeof(LaneType) * 8; + static constexpr bool valid_bit_width = (sizeof(LaneType) == sizeof(u8) && BitWidth >= 1 && BitWidth <= 8) || + (sizeof(LaneType) == sizeof(u16) && BitWidth >= 9 && BitWidth <= 16) || + (sizeof(LaneType) == sizeof(u32) && BitWidth >= 17 && BitWidth <= 24); + static_assert(valid_bit_width, "pack_table BitWidth is outside the permitted range for LaneType"); + + static constexpr usize element_count = VectorBytes / sizeof(LaneType); + static_assert(element_count <= 64, "pack_table supports at most 64 lanes because active masks use u64"); + static_assert(element_count == 0 || element_count - 1 <= static_cast(std::numeric_limits::max()), + "pack_table permute indices must be representable by LaneType"); + static_assert(std::numeric_limits::lowest() <= -1 && static_cast(-1) < 0, + "pack_table LaneType must represent -1 as the inactive-entry sentinel"); + + static constexpr usize contributor_count = BitWidth > 0 ? (lane_bits + BitWidth - 1) / BitWidth : 0; + static_assert(contributor_count > 0, "pack_table must have at least one primary contributor"); + static constexpr u8 bit_width = BitWidth; + + alignas(64) static constexpr auto primary_permutes = + internal::make_primary_permutes(std::make_index_sequence{}); + alignas(64) static constexpr auto primary_shifts = + internal::make_primary_shifts(std::make_index_sequence{}); + alignas(64) static constexpr auto spill_permute = internal::make_spill_permute(); + alignas(64) static constexpr auto spill_shift = internal::make_spill_shift(); + static constexpr auto primary_masks = internal::make_primary_masks(primary_permutes); + static constexpr u64 spill_mask = internal::make_active_mask(spill_permute); +}; +} // namespace pernix::detail + +#endif // PERNIX_PACK_TABLES_H diff --git a/include/pernix/detail/unpack_tables.h b/include/pernix/detail/unpack_tables.h new file mode 100644 index 0000000..eb66927 --- /dev/null +++ b/include/pernix/detail/unpack_tables.h @@ -0,0 +1,162 @@ +#ifndef PERNIX_UNPACK_TABLES_H +#define PERNIX_UNPACK_TABLES_H + +#include + +#include +#include +#include +#include +#include + +namespace pernix::detail { +inline constexpr u8 inactive_lane = 0xff; + +namespace internal { +template +constexpr bool table_indices_are_valid(const std::array& table) { + return std::ranges::all_of(table, [](const u8 index) { return index == inactive_lane || index < Elements; }); +} + +template +constexpr std::array make_primary_permute() { + static_assert(Elements % IntWidth == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (usize entry = 0; entry < Elements / IntWidth; ++entry) { + const usize bit_start = entry * BitWidth; + const usize first_byte = bit_start / 8; + const usize base = entry * IntWidth; + + for (usize lane_byte = 0; lane_byte < IntWidth; ++lane_byte) { + table[base + lane_byte] = static_cast(first_byte + lane_byte); + } + } + + return table; +} + +template +constexpr std::array make_spill_permute() { + static_assert(Elements % IntWidth == 0); + + std::array table{}; + table.fill(inactive_lane); + + for (usize entry = 0; entry < Elements / IntWidth; ++entry) { + const usize bit_start = entry * BitWidth; + const usize first_byte = bit_start / 8; + const usize bit_offset = bit_start % 8; + const usize base = entry * IntWidth; + + if (bit_offset + BitWidth > IntWidth * 8) { + table[base] = static_cast(first_byte + IntWidth); + } + } + + return table; +} + +template +constexpr std::array make_shift_right_magnitude() { + std::array table{}; + + for (usize entry = 0; entry < Elements; ++entry) { + const usize bit_offset = (entry * BitWidth) % 8; + table[entry] = static_cast(bit_offset); + } + + return table; +} + +template +constexpr std::array make_shift_right() { + auto table = make_shift_right_magnitude(); + for (ShiftType& shift : table) { + shift = -shift; + } + return table; +} + +template +constexpr std::array make_shift_left_for_spill() { + std::array table{}; + + for (usize entry = 0; entry < Elements; ++entry) { + const usize bit_offset = (entry * BitWidth) % 8; + const bool spills = bit_offset + BitWidth > IntWidth * 8; + table[entry] = spills ? static_cast(IntWidth * 8 - bit_offset) : 0; + } + + return table; +} +} // namespace internal + +template +struct unpack_table; + +template + requires std::signed_integral && (sizeof(ShiftType) == sizeof(u8)) && (BitWidth >= 1 && BitWidth <= 8) && (VectorBytes > 0) +struct unpack_table { + static constexpr usize shift_elements = VectorBytes / sizeof(ShiftType); + static constexpr u8 bit_width = BitWidth; + + alignas(64) static constexpr std::array primary_permute = + internal::make_primary_permute(); + alignas(64) static constexpr std::array spill_permute = + internal::make_spill_permute(); + alignas(64) static constexpr std::array right_shift = + internal::make_shift_right(); + alignas(64) static constexpr std::array right_shift_magnitude = + internal::make_shift_right_magnitude(); + alignas(64) static constexpr std::array left_shift_for_spill = + internal::make_shift_left_for_spill(); + static constexpr bool has_spill = std::ranges::any_of(left_shift_for_spill, [](const ShiftType shift) { return shift != 0; }); + + static_assert(internal::table_indices_are_valid(primary_permute)); + static_assert(internal::table_indices_are_valid(spill_permute)); +}; + +template + requires std::signed_integral && (sizeof(ShiftType) == sizeof(u16)) && (BitWidth >= 9 && BitWidth <= 16) && (VectorBytes > 0) +struct unpack_table { + static constexpr usize shift_elements = VectorBytes / sizeof(ShiftType); + static constexpr u8 bit_width = BitWidth; + + alignas(64) static constexpr std::array primary_permute = + internal::make_primary_permute(); + alignas(64) static constexpr std::array spill_permute = + internal::make_spill_permute(); + alignas(64) static constexpr std::array right_shift = + internal::make_shift_right(); + alignas(64) static constexpr std::array right_shift_magnitude = + internal::make_shift_right_magnitude(); + alignas(64) static constexpr std::array left_shift_for_spill = + internal::make_shift_left_for_spill(); + static constexpr bool has_spill = std::ranges::any_of(left_shift_for_spill, [](const ShiftType shift) { return shift != 0; }); + + static_assert(internal::table_indices_are_valid(primary_permute)); + static_assert(internal::table_indices_are_valid(spill_permute)); +}; + +template + requires std::signed_integral && (sizeof(ShiftType) == sizeof(u32)) && (BitWidth >= 17 && BitWidth <= 24) && + (VectorBytes > 0) +struct unpack_table { + static constexpr usize shift_elements = VectorBytes / sizeof(ShiftType); + static constexpr u8 bit_width = BitWidth; + + alignas(64) static constexpr std::array primary_permute = + internal::make_primary_permute(); + alignas(64) static constexpr std::array right_shift = + internal::make_shift_right(); + alignas(64) static constexpr std::array right_shift_magnitude = + internal::make_shift_right_magnitude(); + + static_assert(internal::table_indices_are_valid(primary_permute)); +}; +} // namespace pernix::detail + +#endif // PERNIX_UNPACK_TABLES_H diff --git a/include/pernix/detection.h b/include/pernix/detection.h deleted file mode 100644 index edecb6c..0000000 --- a/include/pernix/detection.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef PERNIX_HELPER_H -#define PERNIX_HELPER_H - -#include - -// Internal capability tiers used to derive the public PERNIX_* feature macros. -#define PERNIX_MACHINE_ID_GENERIC 0 -#define PERNIX_MACHINE_ID_V2 1 -#define PERNIX_MACHINE_ID_V3 2 -#define PERNIX_MACHINE_ID_V4 3 -#define PERNIX_MACHINE_ID_V4_VBMI 4 - -// Map the compiler's enabled ISA set to the highest supported Pernix target level. -#if (__SSE3__ && __SSE4_1__ && __SSE4_2__) -#if (__AVX__ && __AVX2__ && __FMA__ && __BMI__ && __BMI2__) -#if (__AVX512BW__ && __AVX512CD__ && __AVX512DQ__ && __AVX512F__ && __AVX512VL__) -#if (__AVX512VBMI__) -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_V4_VBMI -#else -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_V4 -#endif - -#else -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_V3 -#endif - -#else -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_V2 -#endif - -#else -#define PERNIX_MACHINE_ID PERNIX_MACHINE_ID_GENERIC -#endif - -// Feature-selection macros consumed by the public headers. -#if (PERNIX_MACHINE_ID >= PERNIX_MACHINE_ID_V2) -#define PERNIX_SSE_ENABLED -#endif -#if (PERNIX_MACHINE_ID >= PERNIX_MACHINE_ID_V3) -#define PERNIX_AVX2_ENABLED -#define PERNIX_BMI2_ENABLED -#endif -#if (PERNIX_MACHINE_ID >= PERNIX_MACHINE_ID_V4) -#define PERNIX_AVX512_ENABLED -#endif -#if (PERNIX_MACHINE_ID >= PERNIX_MACHINE_ID_V4_VBMI) -#define PERNIX_AVX512_VBMI_ENABLED -#endif - -#ifdef PERNIX_USE_SIMDE -#define PERNIX_SSE_ENABLED -#define PERNIX_AVX2_ENABLED -#define PERNIX_BMI2_ENABLED -#define PERNIX_AVX512_ENABLED -#define PERNIX_AVX512_VBMI_ENABLED -#endif - -// Allow build systems or tests to force lower-tier implementations. -#ifdef PERNIX_DISABLE_AVX512 -#undef PERNIX_AVX512_ENABLED -#undef PERNIX_AVX512_VBMI_ENABLED -#endif -#ifdef PERNIX_DISABLE_AVX2 -#undef PERNIX_AVX2_ENABLED -#endif -#ifdef PERNIX_DISABLE_BMI2 -#undef PERNIX_BMI2_ENABLED -#endif - -#endif // PERNIX_HELPER_H diff --git a/include/pernix/dispatch/cpu_features.h b/include/pernix/dispatch/cpu_features.h new file mode 100644 index 0000000..4d6e866 --- /dev/null +++ b/include/pernix/dispatch/cpu_features.h @@ -0,0 +1,118 @@ +#ifndef PERNIX_CPU_FEATURES_H +#define PERNIX_CPU_FEATURES_H + +#include + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386) || defined(_M_IX86) +#if defined(_MSC_VER) +#include +#else +#include +#include +#endif +#endif + +#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) +#include +#ifndef HWCAP_SVE +#define HWCAP_SVE (1 << 22) +#endif +#ifndef HWCAP2_SVE2 +#define HWCAP2_SVE2 (1 << 1) +#endif +#endif + +namespace pernix::internal { +struct CpuFeatures { + bool avx2 = false; + bool bmi2 = false; + bool avx512f = false; + bool avx512dq = false; + bool avx512bw = false; + bool avx512vl = false; + bool avx512vbmi = false; + bool neon = false; + bool sve = false; + bool sve2 = false; +}; + +#if defined(PERNIX_BUILD_LIB) +CpuFeatures detect_cpu_features(); +CpuFeatures get_cached_cpu_features(); +#else +inline CpuFeatures detect_cpu_features() { + CpuFeatures features{}; + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386) || defined(_M_IX86) + int regs[4]{}; + +#if defined(_MSC_VER) + __cpuidex(regs, 1, 0); + const auto xgetbv = [](const unsigned int index) -> u64 { + return _xgetbv(index); + }; +#else + __cpuid_count(1, 0, regs[0], regs[1], regs[2], regs[3]); + const auto xgetbv = [](const unsigned int index) -> u64 { + u32 eax = 0; + u32 edx = 0; + __asm__ volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(index)); + return static_cast(eax) | (static_cast(edx) << 32U); + }; +#endif + + const bool osxsave = (regs[2] & (1 << 27)) != 0; + const bool avx = (regs[2] & (1 << 28)) != 0; + if (!osxsave || !avx) { + return features; + } + + const u64 xcr0 = xgetbv(0); + const bool xmm_enabled = (xcr0 & 0x2) != 0; + const bool ymm_enabled = (xcr0 & 0x4) != 0; + const bool zmm_enabled = (xcr0 & 0x20) != 0 && (xcr0 & 0x40) != 0 && (xcr0 & 0x80) != 0; + if (!xmm_enabled || !ymm_enabled) { + return features; + } + +#if defined(_MSC_VER) + __cpuidex(regs, 7, 0); +#else + __cpuid_count(7, 0, regs[0], regs[1], regs[2], regs[3]); +#endif + + features.avx2 = (regs[1] & (1 << 5)) != 0; + features.bmi2 = (regs[1] & (1 << 8)) != 0; + if (zmm_enabled) { + features.avx512f = (regs[1] & (1 << 16)) != 0; + features.avx512dq = (regs[1] & (1 << 29)) != 0; + features.avx512bw = (regs[1] & (1 << 30)) != 0; + features.avx512vl = (regs[1] & (1 << 31)) != 0; + features.avx512vbmi = (regs[2] & (1 << 1)) != 0; + } +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) + features.neon = true; +#elif defined(__ARM_NEON) || defined(__ARM_NEON__) + features.neon = true; +#endif + +#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) + const unsigned long hwcap = getauxval(AT_HWCAP); + const unsigned long hwcap2 = getauxval(AT_HWCAP2); + features.sve = (hwcap & HWCAP_SVE) != 0; + features.sve2 = (hwcap2 & HWCAP2_SVE2) != 0; +#endif + + return features; +} + +inline CpuFeatures get_cached_cpu_features() { + static const CpuFeatures features = detect_cpu_features(); + return features; +} +#endif +} // namespace pernix::internal + +#endif // PERNIX_CPU_FEATURES_H diff --git a/include/pernix/dispatch/kernel.h b/include/pernix/dispatch/kernel.h new file mode 100644 index 0000000..aafdab1 --- /dev/null +++ b/include/pernix/dispatch/kernel.h @@ -0,0 +1,25 @@ +#ifndef PERNIX_KERNEL_H +#define PERNIX_KERNEL_H + +#include + +#include + +namespace pernix::internal { +using KernelBlockF32Func = i32 (*)(const void*, f32, void*); +using KernelBlocksF32Func = i32 (*)(const void*, f32, void*, u32); +using KernelBlockF64Func = i32 (*)(const void*, f64, void*); +using KernelBlocksF64Func = i32 (*)(const void*, f64, void*, u32); + +template +struct Kernel { + std::string_view name; + FuncType func; + + explicit operator bool() const noexcept { return func != nullptr; } + + Kernel(const std::string_view kernel_name, FuncType kernel_func) : name(kernel_name), func(kernel_func) {} +}; +} // namespace pernix::internal + +#endif // PERNIX_KERNEL_H diff --git a/include/pernix/dispatch/select.h b/include/pernix/dispatch/select.h new file mode 100644 index 0000000..b3c8334 --- /dev/null +++ b/include/pernix/dispatch/select.h @@ -0,0 +1,240 @@ +#ifndef PERNIX_SELECT_H +#define PERNIX_SELECT_H + +#include +#include + +#if !defined(PERNIX_BUILD_X86_AVX2) && !defined(PERNIX_DISABLE_AVX2) +#if defined(PERNIX_USE_SIMDE) || defined(__AVX2__) +#define PERNIX_BUILD_X86_AVX2 1 +#endif +#endif + +#if !defined(PERNIX_BUILD_X86_BMI2) && !defined(PERNIX_DISABLE_BMI2) +#if defined(PERNIX_USE_SIMDE) || (defined(__AVX2__) && defined(__BMI2__)) +#define PERNIX_BUILD_X86_BMI2 1 +#endif +#endif + +#if !defined(PERNIX_BUILD_X86_AVX512_VBMI) && !defined(PERNIX_DISABLE_AVX512) +#if defined(PERNIX_USE_SIMDE) || \ + (defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && defined(__AVX512VL__) && defined(__AVX512VBMI__)) +#define PERNIX_BUILD_X86_AVX512_VBMI 1 +#endif +#endif + +#if !defined(PERNIX_BUILD_ARM64_NEON) +#if defined(__aarch64__) || defined(_M_ARM64) || defined(__ARM_NEON) || defined(__ARM_NEON__) +#define PERNIX_BUILD_ARM64_NEON 1 +#endif +#endif + +#if !defined(PERNIX_BUILD_ARM64_SVE2) +#if defined(__ARM_FEATURE_SVE2) || defined(PERNIX_USE_SIMDE) +#define PERNIX_BUILD_ARM64_SVE2 1 +#endif +#endif + +namespace pernix::internal { +Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size); + +Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size); + +Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size); + +Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size); + +Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_fallback_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_fallback_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_fallback_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_fallback_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_scalar_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_fallback_scalar_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_fallback_scalar_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_fallback_scalar_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_fallback_scalar_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_scalar_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_scalar_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_scalar_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_stdpar_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_fallback_stdpar_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_fallback_stdpar_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_fallback_stdpar_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_fallback_stdpar_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_stdpar_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_stdpar_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_stdpar_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_simd_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_fallback_simd_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_fallback_simd_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_fallback_simd_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_fallback_simd_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_simd_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_simd_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_fallback_simd_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +#if defined(PERNIX_BUILD_X86_AVX2) + +Kernel select_avx2_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_avx2_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_avx2_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_avx2_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_avx2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_avx2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_avx2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_avx2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + +Kernel select_bmi2_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_bmi2_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_bmi2_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_bmi2_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_bmi2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_bmi2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_bmi2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_bmi2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + +Kernel select_avx512vbmi_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_avx512vbmi_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_avx512vbmi_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_avx512vbmi_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_avx512vbmi_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_avx512vbmi_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_avx512vbmi_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_avx512vbmi_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + +Kernel select_neon_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_neon_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_neon_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_neon_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_neon_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_neon_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_neon_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_neon_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + +Kernel select_sve2_compress_block_f32(u8 bit_width, u32 block_size); + +Kernel select_sve2_compress_blocks_f32(u8 bit_width, u32 block_size); + +Kernel select_sve2_compress_block_f64(u8 bit_width, u32 block_size); + +Kernel select_sve2_compress_blocks_f64(u8 bit_width, u32 block_size); + +Kernel select_sve2_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_sve2_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_sve2_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values); + +Kernel select_sve2_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values); + +#endif +} // namespace pernix::internal + +#if !defined(PERNIX_BUILD_LIB) +#include +#endif + +#endif // PERNIX_SELECT_H diff --git a/include/pernix/dispatch/select_impl.h b/include/pernix/dispatch/select_impl.h new file mode 100644 index 0000000..8737320 --- /dev/null +++ b/include/pernix/dispatch/select_impl.h @@ -0,0 +1,3584 @@ +#ifndef PERNIX_SELECT_IMPL_H +#define PERNIX_SELECT_IMPL_H + +#include +#include +#include +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +#include +#include +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) +#include +#include +#endif +#if defined(PERNIX_BUILD_X86_AVX2) +#include +#include +#endif +#if defined(PERNIX_BUILD_X86_BMI2) +#include +#include +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) +#include +#include +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) +#include +#include +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) +#include +#include +#endif + +namespace pernix::internal { +inline bool is_optional_fallback_backend(const Backend backend) { + switch (backend) { + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + default: + return false; + } +} + +inline bool is_compiled_backend(const Backend backend) { + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + return true; +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return true; +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return true; +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return true; +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return true; +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return true; +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return true; +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return true; +#endif + default: + return false; + } +} + +inline bool is_backend_supported_on_machine(const Backend backend) { + const CpuFeatures features = get_cached_cpu_features(); + + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + case Backend::X86Avx2: + return features.avx2; + case Backend::X86Bmi2: + return features.avx2 && features.bmi2; + case Backend::X86Avx512Vbmi: + return features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi; + case Backend::Arm64Neon: + return features.neon; + case Backend::Arm64Sve: + return features.sve2; + default: + return false; + } +} + +inline Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f32(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +inline Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f32(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +inline Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f64(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +inline Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f64(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +inline Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +inline Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +inline Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +inline Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +inline Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_block_f32(bit_width, block_size); +} + +inline Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_blocks_f32(bit_width, block_size); +} + +inline Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_block_f64(bit_width, block_size); +} + +inline Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_blocks_f64(bit_width, block_size); +} + +inline Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +} + +inline Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +} + +inline Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +} + +inline Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +} +} // namespace pernix::internal + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("fallback", &compress_block_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("fallback", &compress_blocks_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("fallback", &compress_block_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("fallback", &compress_blocks_fallback) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +inline Kernel select_fallback_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_scalar_compress_block_f32(const u8 bit_width, const u32 block_size) { + return select_fallback_compress_block_f32(bit_width, block_size); +} + +inline Kernel select_fallback_scalar_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + return select_fallback_compress_blocks_f32(bit_width, block_size); +} + +inline Kernel select_fallback_scalar_compress_block_f64(const u8 bit_width, const u32 block_size) { + return select_fallback_compress_block_f64(bit_width, block_size); +} + +inline Kernel select_fallback_scalar_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + return select_fallback_compress_blocks_f64(bit_width, block_size); +} + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +inline Kernel select_fallback_stdpar_compress_block_f32(const u8 bit_width, const u32 block_size) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_stdpar_compress_blocks_f32(const u8 bit_width, const u32 block_size) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_stdpar_compress_block_f64(const u8 bit_width, const u32 block_size) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_stdpar_compress_blocks_f64(const u8 bit_width, const u32 block_size) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_simd_compress_block_f32(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + return {"fallback_simd", &compress_block_fallback_simd<1, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_simd_compress_blocks_f32(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + return {"fallback_simd", &compress_blocks_fallback_simd<1, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_simd_compress_block_f64(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + return {"fallback_simd", &compress_block_fallback_simd<1, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_simd_compress_blocks_f64(const u8, const u32) { +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + return {"fallback_simd", &compress_blocks_fallback_simd<1, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ + break + +inline Kernel select_fallback_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"fallback", nullptr}; + } +} + +inline Kernel select_fallback_scalar_decompress_block_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +} + +inline Kernel select_fallback_scalar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +} + +inline Kernel select_fallback_scalar_decompress_block_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +} + +inline Kernel select_fallback_scalar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +} + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } \ + break + +inline Kernel select_fallback_stdpar_decompress_block_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_stdpar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, + const bool sign_values) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_stdpar_decompress_block_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_stdpar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, + const bool sign_values) { +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_simd_decompress_block_f32(const u8, const u32, const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + if (sign_values) { + return {"fallback_simd", &decompress_block_fallback_simd<1, true, 64, f32>}; + } + return {"fallback_simd", &decompress_block_fallback_simd<1, false, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_simd_decompress_blocks_f32(const u8, const u32, const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + if (sign_values) { + return {"fallback_simd", &decompress_blocks_fallback_simd<1, true, 64, f32>}; + } + return {"fallback_simd", &decompress_blocks_fallback_simd<1, false, 64, f32>}; +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_simd_decompress_block_f64(const u8, const u32, const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + if (sign_values) { + return {"fallback_simd", &decompress_block_fallback_simd<1, true, 64, f64>}; + } + return {"fallback_simd", &decompress_block_fallback_simd<1, false, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +inline Kernel select_fallback_simd_decompress_blocks_f64(const u8, const u32, const bool sign_values) { + static_cast(sign_values); +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + if (sign_values) { + return {"fallback_simd", &decompress_blocks_fallback_simd<1, true, 64, f64>}; + } + return {"fallback_simd", &decompress_blocks_fallback_simd<1, false, 64, f64>}; +#else + return {"unsupported_implementation", nullptr}; +#endif +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal +#if defined(PERNIX_BUILD_X86_AVX2) + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("avx2", &mm256_compress_block_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("avx2", &mm256_compress_blocks_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("avx2", &mm256_compress_block_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("avx2", &mm256_compress_blocks_avx2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } + +inline Kernel select_avx2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +inline Kernel select_avx2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +inline Kernel select_avx2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } +} + +inline Kernel select_avx2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ + break + +inline Kernel select_avx2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +inline Kernel select_avx2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"avx2", nullptr}; + } +} + +inline Kernel select_avx2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } +} + +inline Kernel select_avx2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"avx2", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("bmi2", &mm256_compress_block_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("bmi2", &mm256_compress_blocks_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("bmi2", &mm256_compress_block_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("bmi2", &mm256_compress_blocks_bmi2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } + +inline Kernel select_bmi2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +inline Kernel select_bmi2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +inline Kernel select_bmi2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +inline Kernel select_bmi2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ + break + +inline Kernel select_bmi2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +inline Kernel select_bmi2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +inline Kernel select_bmi2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +inline Kernel select_bmi2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +inline Kernel select_avx512vbmi_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +inline Kernel select_avx512vbmi_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +inline Kernel select_avx512vbmi_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +inline Kernel select_avx512vbmi_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ + break + +inline Kernel select_avx512vbmi_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +inline Kernel select_avx512vbmi_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +inline Kernel select_avx512vbmi_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +inline Kernel select_avx512vbmi_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + +namespace pernix::internal { +inline Kernel select_neon_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} + +inline Kernel select_neon_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} + +inline Kernel select_neon_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} + +inline Kernel select_neon_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} +} // namespace pernix::internal + +using pernix::arm64::neon::neon_decompress_block; +using pernix::arm64::neon::neon_decompress_blocks; + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ + break + +inline Kernel select_neon_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"neon", nullptr}; + } +} + +inline Kernel select_neon_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"neon", nullptr}; + } +} + +inline Kernel select_neon_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"neon", nullptr}; + } +} + +inline Kernel select_neon_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"neon", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + +namespace pernix::internal { +inline Kernel select_sve2_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} + +inline Kernel select_sve2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} + +inline Kernel select_sve2_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} + +inline Kernel select_sve2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} +} // namespace pernix::internal + +using pernix::arm64::sve2::sve2_decompress_block; +using pernix::arm64::sve2::sve2_decompress_blocks; + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ + break + +inline Kernel select_sve2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"sve2", nullptr}; + } +} + +inline Kernel select_sve2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"sve2", nullptr}; + } +} + +inline Kernel select_sve2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"sve2", nullptr}; + } +} + +inline Kernel select_sve2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"sve2", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal +#endif + +#endif // PERNIX_SELECT_IMPL_H diff --git a/include/pernix/fallback/common.h b/include/pernix/fallback/common.h new file mode 100644 index 0000000..46d44d2 --- /dev/null +++ b/include/pernix/fallback/common.h @@ -0,0 +1,52 @@ +#ifndef PERNIX_FALLBACK_COMMON_H +#define PERNIX_FALLBACK_COMMON_H + +#include + +#include +#include +#include + +namespace pernix::internal { + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && std::is_integral_v) +__always_inline T quantize_clamped(const FloatT input, const FloatT scale) { + constexpr T min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + + constexpr T max_value = BIT_WIDTH == 1 ? 1 : (1 << (BIT_WIDTH - 1)) - 1; + + const FloatT scaled = input * scale; + + if (std::isnan(scaled)) [[unlikely]] { + return 0; + } + + const FloatT clamped = std::clamp(scaled, static_cast(min_value), static_cast(max_value)); + + return static_cast(std::lrint(clamped)); +} + +template + requires(std::is_integral_v && std::is_floating_point_v) +__always_inline FloatT dequantize(const T input, const FloatT scale) { + return static_cast(input) * scale; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__always_inline constexpr i32 sign_extend(const u32 value) noexcept { + if constexpr (BIT_WIDTH == 1) { + return static_cast(value & 1); + } + + constexpr u32 sign_bit = 1 << (BIT_WIDTH - 1); + constexpr u32 mask = (1 << BIT_WIDTH) - 1; + + const u32 masked = value & mask; + + return static_cast(static_cast(masked ^ sign_bit) - static_cast(sign_bit)); +} +} // namespace pernix::internal + +#endif // PERNIX_FALLBACK_COMMON_H diff --git a/include/pernix/fallback/compression.h b/include/pernix/fallback/compression.h deleted file mode 100644 index 5b2780b..0000000 --- a/include/pernix/fallback/compression.h +++ /dev/null @@ -1,306 +0,0 @@ -#ifndef PERNIX_FALLBACK_COMPRESSION_H -#define PERNIX_FALLBACK_COMPRESSION_H - -#include - -#include -#include -#include -#include -#include -#include - -namespace pernix { -namespace internal { -/** - * @brief Quantize a single float value to int32_t using the provided scale. - * - * @param input input float value to be quantized. - * @param scale scaling factor used during quantization. - * @return int32_t quantized integer value. - */ -__always_inline int32_t quantize_ps_epi32(const float input, const float scale) { - return static_cast(std::lroundf(input * scale)); -} - -/** - * @brief Quantize a single double value to int64_t using the provided scale. - * - * @param input input double value to be quantized. - * @param scale scaling factor used during quantization. - * @return int64_t quantized integer value. - */ -__always_inline int64_t quantize_pd_epi64(const double_t input, const double_t scale) { - return std::llround(input * scale); -} - -/** - * @brief Quantize and clamp without narrowing through an out-of-range integer type. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) -__always_inline int32_t quantize_clamped(const T input, const T scale) { - constexpr int64_t min_value = BIT_WIDTH == 1 ? 0 : -(int64_t{1} << (BIT_WIDTH - 1)); - constexpr int64_t max_value = BIT_WIDTH == 1 ? 1 : ((int64_t{1} << (BIT_WIDTH - 1)) - 1); - - const long double scaled = static_cast(input) * static_cast(scale); - if (std::isnan(scaled)) { - return 0; - } - if (scaled <= static_cast(min_value)) { - return static_cast(min_value); - } - if (scaled >= static_cast(max_value)) { - return static_cast(max_value); - } - - return static_cast(std::llround(scaled)); -} - -/** - * @brief Clamp a signed quantized value to the representable range of BIT_WIDTH bits. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline int32_t clamp_signed_quantized(const int64_t value) { - if constexpr (BIT_WIDTH == 1) { - // 1-bit fallback is treated as binary quantization (0/1). - return static_cast(std::clamp(value, 0, 1)); - } - - constexpr int32_t min_value = -(1 << (BIT_WIDTH - 1)); - constexpr int32_t max_value = (1 << (BIT_WIDTH - 1)) - 1; - return static_cast(std::clamp(value, min_value, max_value)); -} - -/** - * @brief Append packed scalar values into an output buffer using the selected - * storage width. - * - * @tparam T unsigned integer type used as the packing word. - * @tparam BIT_WIDTH bit width per value in the packed representation. - * @param input vector of quantized values to pack. - * @param bit_offset starting bit offset in the destination buffer. - * @param destination pointer to the output buffer. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) -void pack_epi32_fallback_inner(const std::vector& input, const uint8_t bit_offset, uint8_t* __restrict__ destination) { - constexpr uint32_t bits_in_type = sizeof(T) * 8; - constexpr uint32_t bitmask = BIT_WIDTH == bits_in_type ? std::numeric_limits::max() : (1U << BIT_WIDTH) - 1U; - - std::size_t idx = 0; - std::size_t bits_in_buffer = bit_offset; - uint64_t buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; - -#pragma GCC unroll 64 - for (uint32_t raw_value : input) { - const uint32_t next_value = raw_value & bitmask; - - buffer |= static_cast(next_value) << bits_in_buffer; - bits_in_buffer += BIT_WIDTH; - - while (bits_in_buffer >= 8) { - destination[idx++] = static_cast(buffer & 0xFFU); - buffer >>= 8; - bits_in_buffer -= 8; - } - } - - if (bits_in_buffer > 0) { - destination[idx] = static_cast(buffer & 0xFFU); - } -} - -/** - * @brief Pack a vector of uint32_t values into a compact byte representation using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * - * @param input vector of uint32_t values to be packed. - * @param destination pointer to the output buffer where packed bytes will be stored. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -void pack_epi32_fallback(const std::vector& input, uint8_t* __restrict__ destination) { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } else { - return internal::pack_epi32_fallback_inner(input, 0, destination); - } -} -} // namespace internal - -/** - * @brief Compress a single 512-bit block using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_block_fallback(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - std::memset(output, 0, BLOCK_SIZE); - - std::vector block_values(elements_per_block); -#pragma GCC unroll 64 - for (uint32_t i = 0; i < elements_per_block; i++) { - const int32_t quantized = internal::quantize_clamped(input[i], scale); - block_values[i] = static_cast(quantized); - } - - internal::pack_epi32_fallback(block_values, output); - return 0; -} - -/** - * @brief Compress a single block of double values using the fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_block_fallback(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - std::memset(output, 0, BLOCK_SIZE); - - std::vector block_values(elements_per_block); -#pragma GCC unroll 32 - for (uint32_t i = 0; i < elements_per_block; i++) { - const int32_t quantized = internal::quantize_clamped(input[i], scale); - block_values[i] = static_cast(quantized); - } - - internal::pack_epi32_fallback(block_values, output); - return 0; -} - -/** - * @brief Compress multiple 512-bit blocks using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_blocks_fallback(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - const float_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - compress_block_fallback(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - - return 0; -} - -/** - * @brief Compress multiple blocks of double values using the fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (default 64 for 512 bits). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of blocks to compress. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int compress_blocks_fallback(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - const double_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - compress_block_fallback(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - return 0; -} -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Compress a single 512-bit block using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - */ -int compress_block_fallback(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single 512-bit block using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - */ -int compress_block_fallback_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress multiple 512-bit blocks using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - */ -int compress_blocks_fallback(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Compress multiple 512-bit blocks using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - */ -int compress_blocks_fallback_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - -#endif // PERNIX_FALLBACK_COMPRESSION_H diff --git a/include/pernix/fallback/decompression.h b/include/pernix/fallback/decompression.h deleted file mode 100644 index c4714e3..0000000 --- a/include/pernix/fallback/decompression.h +++ /dev/null @@ -1,286 +0,0 @@ -#ifndef PERNIX_FALLBACK_DECOMPRESSION_H -#define PERNIX_FALLBACK_DECOMPRESSION_H - -#include - -#include -#include -#include -#include - -namespace pernix { -namespace internal { -/** - * @brief Dequantize a single int32_t value to float using the provided scale. - * - * @param input input int32_t value to be dequantized. - * @param scale scaling factor used during quantization. - * @return float dequantized float value. - */ -__always_inline float dequantize_epi32(const int32_t input, const float scale) { - return static_cast(input) * scale; -} - -/** - * @brief Dequantize a single int64_t value to double using the provided scale. - * - * @param input input int64_t value to be dequantized. - * @param scale scaling factor used during quantization. - * @return double_t dequantized double value. - */ -__always_inline double_t dequantize_epi64(const int64_t input, const double_t scale) { - return static_cast(input) * scale; -} - -/** - * @brief Sign-extend a packed integer value stored in the low bits of a 32-bit word. - * - * @tparam BIT_WIDTH number of significant bits in the encoded value. - * @param value unsigned packed value. - * @return int32_t sign-extended value. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline auto sign_extend(const uint32_t value) -> int32_t { - if constexpr (BIT_WIDTH == 1) { - return static_cast(value & 1U); - } - - constexpr uint32_t sign_bit = uint32_t{1} << (BIT_WIDTH - 1); - constexpr uint32_t mask = (uint32_t{1} << BIT_WIDTH) - 1; - const uint32_t masked = value & mask; - return static_cast((static_cast(masked ^ sign_bit)) - static_cast(sign_bit)); -} - -/** - * @brief Unpack bit-packed values from a typed input span into signed 32-bit integers. - * - * @tparam T unsigned integer type used to read the source buffer. - * @tparam BIT_WIDTH bit width per packed value. - * @tparam SIGN_VALUES whether to sign-extend unpacked values. - * @param input pointer to the typed packed input buffer. - * @param bit_offset starting bit offset in the first input word. - * @param elements number of values to unpack. - * @return std::vector unpacked values. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v) -__always_inline auto unpack_epi32_fallback_inner(const uint8_t* __restrict__ input, const uint8_t bit_offset, const std::size_t elements) - -> std::vector { - constexpr uint32_t bits_in_type = sizeof(T) * 8; - constexpr uint32_t bitmask = BIT_WIDTH == bits_in_type ? std::numeric_limits::max() : (1U << BIT_WIDTH) - 1U; - - std::vector output(elements); - - std::size_t idx = 0; - uint8_t bits_in_buffer = 8 - bit_offset; - uint64_t buffer = static_cast(input[idx++]) >> bit_offset; - -#pragma GCC unroll 64 - for (uint32_t i = 0; i < elements; i++) { - while (BIT_WIDTH > bits_in_buffer) { - const auto next_value = static_cast(input[idx++]) << bits_in_buffer; - buffer |= next_value; - bits_in_buffer += 8; - } - - const uint32_t raw_value = static_cast(buffer & bitmask); - if constexpr (SIGN_VALUES) { - output[i] = sign_extend(raw_value); - } else { - output[i] = static_cast(raw_value); - } - - buffer >>= BIT_WIDTH; - bits_in_buffer -= BIT_WIDTH; - } - - return output; -} - -/** - * @brief Unpack packed int32_t values from the input buffer using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the packed data. - * @param elements number of elements to unpack. - * @return std::vector unpacked int32_t values. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__always_inline auto unpack_epi32_fallback(const uint8_t* __restrict__ input, const std::size_t elements) -> std::vector { - if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - return unpack_epi32_fallback_inner(input, 0, elements); - } else { - return unpack_epi32_fallback_inner(input, 0, elements); - } -} -} // namespace internal - -/** - * @brief Decompress a single 512\-bit block using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_block_fallback(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - const std::vector block_values = internal::unpack_epi32_fallback(input, elements_per_block); - -#pragma GCC unroll 512 - for (uint32_t i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi32(block_values[i], scale); - } - - return 0; -} - -/** - * @brief Decompress a single block to double values using the fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_block_fallback(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - - const std::vector block_values = internal::unpack_epi32_fallback(input, elements_per_block); - -#pragma GCC unroll 512 - for (uint32_t i = 0; i < elements_per_block; i++) { - output[i] = internal::dequantize_epi64(block_values[i], scale); - } - - return 0; -} - -/** - * @brief Decompress multiple 512\-bit blocks using fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -int decompress_blocks_fallback(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; -} - -/** - * @brief Decompress multiple blocks to double values using the fallback scalar implementation. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam SIGN_VALUES whether the values are signed or unsigned. - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @param blocks number of blocks to decompress. - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -int decompress_blocks_fallback(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - decompress_block_fallback(block_input, scale, block_output); - block_input += BLOCK_SIZE; - block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; - } - - return 0; -} -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Decompress a single 512-bit block using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - */ -int decompress_block_fallback(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single 512-bit block using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - */ -int decompress_block_fallback_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -/** - * @brief Decompress multiple 512-bit blocks using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - */ -int decompress_blocks_fallback(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Decompress multiple 512-bit blocks using fallback scalar implementation. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - */ -int decompress_blocks_fallback_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - -#endif // PERNIX_FALLBACK_DECOMPRESSION_H diff --git a/include/pernix/fallback/scalar_compression.h b/include/pernix/fallback/scalar_compression.h new file mode 100644 index 0000000..cbbe968 --- /dev/null +++ b/include/pernix/fallback/scalar_compression.h @@ -0,0 +1,108 @@ +#ifndef PERNIX_FALLBACK_SCALAR_COMPRESSION_H +#define PERNIX_FALLBACK_SCALAR_COMPRESSION_H + +#include +#include + +#include +#include +#include +#include +#include + +namespace pernix { +namespace internal { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v && std::is_floating_point_v) +__always_inline void quantize_and_pack_fallback_inner(const std::span input, const FloatT scale, const u32 elements, + const u8 bit_offset, std::span destination) { + constexpr u32 bits_in_type = sizeof(T) * 8; + constexpr T bitmask = BIT_WIDTH == bits_in_type ? std::numeric_limits::max() : static_cast((1U << BIT_WIDTH) - 1U); + + usize idx = 0; + usize bits_in_buffer = bit_offset; + u64 buffer = bit_offset ? static_cast(destination[0] & ((1U << bit_offset) - 1U)) : 0; + +#pragma GCC unroll 512 + for (usize i = 0; i < elements; i++) { + const i32 quantized = quantize_clamped(input[i], scale); + const u32 next_value = static_cast(quantized) & static_cast(bitmask); + + buffer |= static_cast(next_value) << bits_in_buffer; + bits_in_buffer += BIT_WIDTH; + + while (bits_in_buffer >= 8) { + destination[idx++] = static_cast(buffer & 0xFFU); + buffer >>= 8; + bits_in_buffer -= 8; + } + } + + if (bits_in_buffer > 0) { + destination[idx] = static_cast(buffer & 0xFFU); + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +__always_inline void quantize_and_pack_fallback(const std::span input, const FloatT scale, const u32 elements, + std::span destination) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); + } else { + quantize_and_pack_fallback_inner(input, scale, elements, 0, destination); + } +} +} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_block_fallback(const std::span input, const FloatT scale, std::span destination) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + std::ranges::fill(destination, 0); + + internal::quantize_and_pack_fallback(input, scale, elements_per_block, destination); + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_block_fallback(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), elements_per_block); + auto output_span = std::span(static_cast(output_ptr), BLOCK_SIZE); + + return compress_block_fallback(input_span, scale, output_span); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_blocks_fallback(const std::span input, const FloatT scale, std::span destination, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + for (usize i = 0; i < blocks; i++) { + const auto block_input = input.subspan(i * elements_per_block, elements_per_block); + auto block_output = destination.subspan(i * BLOCK_SIZE, BLOCK_SIZE); + compress_block_fallback(block_input, scale, block_output); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_blocks_fallback(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), elements_per_block * blocks); + auto output_span = std::span(static_cast(output_ptr), blocks * BLOCK_SIZE); + + return compress_blocks_fallback(input_span, scale, output_span, blocks); +} +} // namespace pernix + +#endif // PERNIX_FALLBACK_SCALAR_COMPRESSION_H diff --git a/include/pernix/fallback/scalar_decompression.h b/include/pernix/fallback/scalar_decompression.h new file mode 100644 index 0000000..126cb40 --- /dev/null +++ b/include/pernix/fallback/scalar_decompression.h @@ -0,0 +1,108 @@ +#ifndef PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H +#define PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H + +#include +#include + +#include +#include +#include + +namespace pernix { +namespace internal { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_integral_v && std::is_unsigned_v && std::is_floating_point_v) +__always_inline void unpack_and_dequantize_fallback_inner(const std::span input, const u8 bit_offset, const u32 elements, + const FloatT scale, std::span output) { + constexpr u32 bits_in_type = sizeof(T) * 8; + constexpr T bitmask = BIT_WIDTH == bits_in_type ? std::numeric_limits::max() : static_cast((1U << BIT_WIDTH) - 1U); + + usize idx = 0; + u8 bits_in_buffer = 8 - bit_offset; + u64 buffer = static_cast(input[idx++]) >> bit_offset; + +#pragma GCC unroll 512 + for (usize i = 0; i < elements; i++) { + while (BIT_WIDTH > bits_in_buffer) { + const auto next_value = static_cast(input[idx++]) << bits_in_buffer; + buffer |= next_value; + bits_in_buffer += 8; + } + + const u32 raw_value = static_cast(buffer & bitmask); + i32 unpacked = 0; + if constexpr (SIGN_VALUES) { + unpacked = sign_extend(raw_value); + } else { + unpacked = static_cast(raw_value); + } + output[i] = static_cast(unpacked) * scale; + + buffer >>= BIT_WIDTH; + bits_in_buffer -= BIT_WIDTH; + } +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +__always_inline void unpack_and_dequantize_fallback(const std::span input, const u32 elements, const FloatT scale, + std::span output) { + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { + unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); + } else { + unpack_and_dequantize_fallback_inner(input, 0, elements, scale, output); + } +} +} // namespace internal + +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_block_fallback(const std::span input, const FloatT scale, std::span output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + internal::unpack_and_dequantize_fallback(input, elements_per_block, scale, output); + return 0; +} + +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_block_fallback(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto input = std::span(static_cast(input_ptr), BLOCK_SIZE); + auto output = std::span(static_cast(output_ptr), elements_per_block); + + return decompress_block_fallback(input, scale, output); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0 && std::is_floating_point_v) +int decompress_blocks_fallback(const std::span input, const FloatT scale, std::span output, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + for (u32 block = 0; block < blocks; block++) { + const auto block_input = input.subspan(block * BLOCK_SIZE, BLOCK_SIZE); + auto block_output = output.subspan(block * elements_per_block, elements_per_block); + + decompress_block_fallback(block_input, scale, block_output); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0 && std::is_floating_point_v) +int decompress_blocks_fallback(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), blocks * BLOCK_SIZE); + auto output_span = std::span(static_cast(output_ptr), elements_per_block * blocks); + + return decompress_blocks_fallback(input_span, scale, output_span, blocks); +} + +} // namespace pernix + +#endif // PERNIX_FALLBACK_SCALAR_DECOMPRESSION_H diff --git a/include/pernix/fallback/simd_compression.h b/include/pernix/fallback/simd_compression.h new file mode 100644 index 0000000..d87e8f6 --- /dev/null +++ b/include/pernix/fallback/simd_compression.h @@ -0,0 +1,47 @@ +#ifndef PERNIX_FALLBACK_SIMD_COMPRESSION_H +#define PERNIX_FALLBACK_SIMD_COMPRESSION_H + +#include + +#include +#include +#include + +#if defined(__has_include) +#if __has_include() +#include +#endif +#endif + +namespace pernix { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int compress_block_fallback_simd(const void* input_ptr, ScaleType scale, void* output_ptr) { +#if defined(__cpp_lib_simd) + using simd_placeholder = std::simd; + [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); +#endif + [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(elements_per_block); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int compress_blocks_fallback_simd(const void* input_ptr, ScaleType scale, void* output_ptr, u32 blocks) { +#if defined(__cpp_lib_simd) + using simd_placeholder = std::simd; + [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); +#endif + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(blocks); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} +} // namespace pernix + +#endif // PERNIX_FALLBACK_SIMD_COMPRESSION_H diff --git a/include/pernix/fallback/simd_decompression.h b/include/pernix/fallback/simd_decompression.h new file mode 100644 index 0000000..a876948 --- /dev/null +++ b/include/pernix/fallback/simd_decompression.h @@ -0,0 +1,51 @@ +#ifndef PERNIX_FALLBACK_SIMD_DECOMPRESSION_H +#define PERNIX_FALLBACK_SIMD_DECOMPRESSION_H + +#include + +#include +#include +#include + +#if defined(__has_include) +#if __has_include() +#include +#endif +#endif + +namespace pernix { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int decompress_block_fallback_simd(const void* input_ptr, ScaleType scale, void* output_ptr) { +#if defined(__cpp_lib_simd) + using simd_placeholder = std::simd; + [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); +#endif + [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; + [[maybe_unused]] constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(sign_values); + static_cast(elements_per_block); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v) +int decompress_blocks_fallback_simd(const void* input_ptr, ScaleType scale, void* output_ptr, u32 blocks) { +#if defined(__cpp_lib_simd) + using simd_placeholder = std::simd; + [[maybe_unused]] constexpr std::size_t simd_lanes = simd_placeholder::size(); +#endif + [[maybe_unused]] constexpr bool sign_values = SIGN_VALUES; + static_cast(input_ptr); + static_cast(scale); + static_cast(output_ptr); + static_cast(blocks); + static_cast(sign_values); + return PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION; +} +} // namespace pernix + +#endif // PERNIX_FALLBACK_SIMD_DECOMPRESSION_H diff --git a/include/pernix/fallback/stdpar_common.h b/include/pernix/fallback/stdpar_common.h new file mode 100644 index 0000000..c61ed8d --- /dev/null +++ b/include/pernix/fallback/stdpar_common.h @@ -0,0 +1,115 @@ +#ifndef PERNIX_FALLBACK_STDPAR_COMMON_H +#define PERNIX_FALLBACK_STDPAR_COMMON_H + +#include + +#include +#include +#if defined(PERNIX_STDPAR_USE_PARALLEL_POLICY) +#include +#endif +#include +#include + +namespace pernix::internal { +inline constexpr u32 stdpar_group_size_v = 8; + +template + requires(GROUP_SIZE >= 1 && GROUP_SIZE <= 8 && BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +inline constexpr usize packed_group_bytes_v = (static_cast(GROUP_SIZE) * BIT_WIDTH + 7U) / 8U; + +template +class counting_iterator { +public: + using iterator_category = std::random_access_iterator_tag; + using iterator_concept = std::random_access_iterator_tag; + using value_type = T; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = T; + + constexpr explicit counting_iterator(T value) noexcept : value_(value) {} + + constexpr T operator*() const noexcept { return value_; } + + constexpr T operator[](difference_type n) const noexcept { return value_ + n; } + + constexpr counting_iterator& operator++() noexcept { + ++value_; + return *this; + } + + constexpr counting_iterator operator++(i32) noexcept { + counting_iterator tmp = *this; + ++(*this); + return tmp; + } + + constexpr counting_iterator& operator--() noexcept { + --value_; + return *this; + } + + constexpr counting_iterator operator--(i32) noexcept { + counting_iterator tmp = *this; + --(*this); + return tmp; + } + + constexpr counting_iterator& operator+=(difference_type n) noexcept { + value_ += static_cast(n); + return *this; + } + + constexpr counting_iterator& operator-=(difference_type n) noexcept { + value_ -= static_cast(n); + return *this; + } + + friend constexpr auto operator+(counting_iterator it, difference_type n) noexcept -> counting_iterator { + it += n; + return it; + } + + friend constexpr auto operator+(difference_type n, counting_iterator it) noexcept -> counting_iterator { + it += n; + return it; + } + + friend constexpr auto operator-(counting_iterator it, difference_type n) noexcept -> counting_iterator { + it -= n; + return it; + } + + friend constexpr difference_type operator-(counting_iterator lhs, counting_iterator rhs) noexcept { + return static_cast(lhs.value_) - static_cast(rhs.value_); + } + + friend constexpr bool operator==(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ == rhs.value_; } + + friend constexpr bool operator!=(counting_iterator lhs, counting_iterator rhs) noexcept { return !(lhs == rhs); } + + friend constexpr bool operator<(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ < rhs.value_; } + + friend constexpr bool operator<=(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ <= rhs.value_; } + + friend constexpr bool operator>(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ > rhs.value_; } + + friend constexpr bool operator>=(counting_iterator lhs, counting_iterator rhs) noexcept { return lhs.value_ >= rhs.value_; } + +private: + T value_; +}; + +template +__always_inline void for_each_index_stdpar(const u32 count, Func&& func) { + auto first = counting_iterator{0}; +#if defined(PERNIX_STDPAR_USE_PARALLEL_POLICY) + std::for_each_n(std::execution::par_unseq, first, count, std::forward(func)); +#else + std::for_each_n(first, count, std::forward(func)); +#endif +} +} // namespace pernix::internal + +#endif // PERNIX_FALLBACK_STDPAR_COMMON_H diff --git a/include/pernix/fallback/stdpar_compression.h b/include/pernix/fallback/stdpar_compression.h new file mode 100644 index 0000000..f6b8283 --- /dev/null +++ b/include/pernix/fallback/stdpar_compression.h @@ -0,0 +1,82 @@ +#ifndef PERNIX_FALLBACK_STDPAR_COMPRESSION_H +#define PERNIX_FALLBACK_STDPAR_COMPRESSION_H + +#include +#include +#include + +#include +#include +#include +#include + +namespace pernix { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_block_fallback_stdpar(const std::span input, const FloatT scale, std::span destination) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + constexpr u32 full_groups = elements_per_block / internal::stdpar_group_size_v; + constexpr u32 remainder = elements_per_block % internal::stdpar_group_size_v; + + std::ranges::fill(destination, 0); + + internal::for_each_index_stdpar(full_groups, [&](const u32 group) { + constexpr u32 group_size = internal::stdpar_group_size_v; + + const auto input_group = input.subspan(static_cast(group) * group_size, group_size); + auto destination_group = destination.subspan(static_cast(group) * BIT_WIDTH, BIT_WIDTH); + + internal::quantize_and_pack_fallback(input_group, scale, group_size, destination_group); + }); + + if constexpr (remainder != 0) { + constexpr usize tail_input_offset = full_groups * internal::stdpar_group_size_v; + constexpr usize tail_output_offset = full_groups * BIT_WIDTH; + constexpr usize tail_bytes = internal::packed_group_bytes_v(remainder), BIT_WIDTH>; + + internal::quantize_and_pack_fallback(input.subspan(tail_input_offset, remainder), scale, remainder, + destination.subspan(tail_output_offset, tail_bytes)); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_block_fallback_stdpar(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), elements_per_block); + auto output_span = std::span(static_cast(output_ptr), BLOCK_SIZE); + + return compress_block_fallback_stdpar(input_span, scale, output_span); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_blocks_fallback_stdpar(const std::span input, const FloatT scale, std::span destination, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + internal::for_each_index_stdpar(blocks, [&](const u32 block) { + const auto block_input = input.subspan(static_cast(block) * elements_per_block, elements_per_block); + auto block_output = destination.subspan(static_cast(block) * BLOCK_SIZE, BLOCK_SIZE); + static_cast(compress_block_fallback_stdpar(block_input, scale, block_output)); + }); + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int compress_blocks_fallback_stdpar(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr, + const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), elements_per_block * blocks); + auto output_span = std::span(static_cast(output_ptr), static_cast(blocks) * BLOCK_SIZE); + + return compress_blocks_fallback_stdpar(input_span, scale, output_span, blocks); +} +} // namespace pernix + +#endif // PERNIX_FALLBACK_STDPAR_COMPRESSION_H diff --git a/include/pernix/fallback/stdpar_decompression.h b/include/pernix/fallback/stdpar_decompression.h new file mode 100644 index 0000000..1b18080 --- /dev/null +++ b/include/pernix/fallback/stdpar_decompression.h @@ -0,0 +1,79 @@ +#ifndef PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H +#define PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H + +#include +#include +#include + +#include +#include +#include + +namespace pernix { +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_block_fallback_stdpar(const std::span input, const FloatT scale, std::span output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + constexpr u32 full_groups = elements_per_block / internal::stdpar_group_size_v; + constexpr u32 remainder = elements_per_block % internal::stdpar_group_size_v; + + internal::for_each_index_stdpar(full_groups, [&](const u32 group) { + constexpr u32 group_size = internal::stdpar_group_size_v; + + const auto input_group = input.subspan(static_cast(group) * BIT_WIDTH, BIT_WIDTH); + auto output_group = output.subspan(static_cast(group) * group_size, group_size); + + internal::unpack_and_dequantize_fallback(input_group, group_size, scale, output_group); + }); + + if constexpr (remainder != 0) { + constexpr usize tail_input_offset = full_groups * BIT_WIDTH; + constexpr usize tail_output_offset = full_groups * internal::stdpar_group_size_v; + constexpr usize tail_bytes = internal::packed_group_bytes_v(remainder), BIT_WIDTH>; + + internal::unpack_and_dequantize_fallback(input.subspan(tail_input_offset, tail_bytes), remainder, scale, + output.subspan(tail_output_offset, remainder)); + } + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_block_fallback_stdpar(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), BLOCK_SIZE); + auto output_span = std::span(static_cast(output_ptr), elements_per_block); + + return decompress_block_fallback_stdpar(input_span, scale, output_span); +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_blocks_fallback_stdpar(const std::span input, const FloatT scale, std::span output, const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + internal::for_each_index_stdpar(blocks, [&](const u32 block) { + const auto block_input = input.subspan(static_cast(block) * BLOCK_SIZE, BLOCK_SIZE); + auto block_output = output.subspan(static_cast(block) * elements_per_block, elements_per_block); + static_cast(decompress_block_fallback_stdpar(block_input, scale, block_output)); + }); + + return 0; +} + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24 && std::is_floating_point_v && BLOCK_SIZE > 0 && BLOCK_SIZE % 32 == 0) +int decompress_blocks_fallback_stdpar(const void* __restrict__ input_ptr, const FloatT scale, void* __restrict__ output_ptr, + const u32 blocks) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8U) / BIT_WIDTH; + + const auto input_span = std::span(static_cast(input_ptr), static_cast(blocks) * BLOCK_SIZE); + auto output_span = std::span(static_cast(output_ptr), elements_per_block * blocks); + + return decompress_blocks_fallback_stdpar(input_span, scale, output_span, blocks); +} +} // namespace pernix + +#endif // PERNIX_FALLBACK_STDPAR_DECOMPRESSION_H diff --git a/include/pernix/pernix.h b/include/pernix/pernix.h index 55133bb..407b3b6 100644 --- a/include/pernix/pernix.h +++ b/include/pernix/pernix.h @@ -1,423 +1,102 @@ #ifndef PERNIX_H #define PERNIX_H -#include +#include -// Include architecture-specific headers based on detected capabilities -// AVX2 -#ifdef PERNIX_AVX2_ENABLED -#include -#include - -// BMI2: Needs AVX2 as well -#ifdef PERNIX_BMI2_ENABLED -#include -#include -#endif // PERNIX_BMI2_ENABLED - -// AVX512 VBMI: Needs AVX2 as well -#ifdef PERNIX_AVX512_VBMI_ENABLED -#include -#include -#endif // PERNIX_AVX512_VBMI_ENABLED - -#endif // PERNIX_AVX2_ENABLED - -// Fallback (non-SIMD) implementations -#include -#include - -namespace pernix { -/** - * @brief Compress a single block of floating-point data into a bit-packed format using the specified bit width and scale. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * - * @return int status code (0 for success). - * - * @note This function will dispatch to the best available implementation based on detected CPU features at compile time. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single block of double-precision values into a bit-packed representation. - * - * @tparam BIT_WIDTH bit width per quantized value (1 to 24). - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the input block. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress multiple blocks of single-precision values. - * - * @tparam BIT_WIDTH bit width per quantized value (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (must be a multiple of 32). - * - * @param input pointer to the first input value. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @param blocks number of consecutive blocks to process. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, uint32_t blocks); - -/** - * @brief Compress multiple blocks of double-precision values. - * - * @tparam BIT_WIDTH bit width per quantized value (1 to 24). - * @tparam BLOCK_SIZE size of each block in bytes (must be a multiple of 32). - * - * @param input pointer to the first input value. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @param blocks number of consecutive blocks to process. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, uint32_t blocks); - -/** - * @brief Decompress a single block of packed values into single-precision values. - * - * @tparam BIT_WIDTH bit width per packed value (1 to 24). - * @tparam SIGN_VALUES true for signed values, false for unsigned values. - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the packed input block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single block of packed values into double-precision values. - * - * @tparam BIT_WIDTH bit width per packed value (1 to 24). - * @tparam SIGN_VALUES true for signed values, false for unsigned values. - * @tparam BLOCK_SIZE size of the block in bytes (must be a multiple of 32). - * - * @param input pointer to the packed input block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -/** - * @brief Decompress multiple blocks of packed values into single-precision values. - * - * @tparam BIT_WIDTH bit width per packed value (1 to 24). - * @tparam SIGN_VALUES true for signed values, false for unsigned values. - * @tparam BLOCK_SIZE size of each block in bytes (must be a multiple of 32). - * - * @param input pointer to the first packed block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @param blocks number of consecutive blocks to process. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); - -/** - * @brief Decompress multiple blocks of packed values into double-precision values. - * - * @tparam BIT_WIDTH bit width per packed value (1 to 24). - * @tparam SIGN_VALUES true for signed values, false for unsigned values. - * @tparam BLOCK_SIZE size of each block in bytes (must be a multiple of 32). - * - * @param input pointer to the first packed block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @param blocks number of consecutive blocks to process. - * - * @return int status code (0 for success). - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, uint32_t blocks); - -// Use the best available implementation based on detected CPU features at compile time. -#ifdef PERNIX_AVX2_ENABLED -#ifdef PERNIX_AVX512_VBMI_ENABLED -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return mm512_compress_block_avx512vbmi(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return mm512_compress_block_avx512vbmi(input, scale, output); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return mm512_compress_blocks_avx512vbmi(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return mm512_compress_blocks_avx512vbmi(input, scale, output, blocks); -} - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return mm512_decompress_block_avx512vbmi(input, scale, output); -} +#ifndef __cplusplus +#include +#endif -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return mm512_decompress_block_avx512vbmi(input, scale, output); -} +#ifdef __cplusplus +extern "C" { +#endif -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - return mm512_decompress_blocks_avx512vbmi(input, scale, output, blocks); -} +typedef enum pernix_status { // NOLINT(cppcoreguidelines-use-enum-class, modernize-use-using) + PERNIX_STATUS_OK = 0, + PERNIX_STATUS_INVALID_ARGUMENT = -1, + PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH = -2, + PERNIX_STATUS_UNSUPPORTED_BACKEND = -3, + PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE = -4, + PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION = -5, +} pernix_status; -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - return mm512_decompress_blocks_avx512vbmi(input, scale, output, blocks); -} -#else -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return mm256_compress_block_avx2(input, scale, output); -} +typedef enum pernix_backend { // NOLINT(cppcoreguidelines-use-enum-class, modernize-use-using) + PERNIX_BACKEND_AUTO = 0, + PERNIX_BACKEND_FALLBACK = 1, + PERNIX_BACKEND_FALLBACK_SCALAR = PERNIX_BACKEND_FALLBACK, + PERNIX_BACKEND_X86_AVX2 = 2, + PERNIX_BACKEND_X86_BMI2 = 3, + PERNIX_BACKEND_X86_AVX512_VBMI = 4, + PERNIX_BACKEND_ARM64_NEON = 5, + PERNIX_BACKEND_ARM64_SVE = 6, + PERNIX_BACKEND_FALLBACK_STDPAR = 7, + PERNIX_BACKEND_FALLBACK_SIMD = 8, +} pernix_backend; -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return mm256_compress_block_avx2(input, scale, output); -} +PERNIX_API u8 pernix_min_bit_width(void); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return mm256_compress_blocks_avx2(input, scale, output, blocks); -} +PERNIX_API u8 pernix_max_bit_width(void); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return mm256_compress_blocks_avx2(input, scale, output, blocks); -} +PERNIX_API bool pernix_is_valid_bit_width(u8 bit_width); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return mm256_decompress_block_avx2(input, scale, output); -} +PERNIX_API bool pernix_is_valid_block_size(u32 block_size); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return mm256_decompress_block_avx2(input, scale, output); -} +PERNIX_API u32 pernix_compressed_block_size(void); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - return mm256_decompress_blocks_avx2(input, scale, output, blocks); -} +PERNIX_API u32 pernix_elements_per_block(u8 bit_width); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - return mm256_decompress_blocks_avx2(input, scale, output, blocks); -} -#endif -#else -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return compress_block_fallback(input, scale, output); -} +PERNIX_API const char* pernix_status_string(pernix_status status); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_block(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return compress_block_fallback(input, scale, output); -} +/* Computes the forward decompression scale from bmax and bit width. */ +PERNIX_API pernix_status pernix_scale_f32(float bmax, u8 bit_width, float* scale); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return compress_blocks_fallback(input, scale, output, blocks); -} +/* Computes the forward decompression scale from bmax and bit width. */ +PERNIX_API pernix_status pernix_scale_f64(double bmax, u8 bit_width, double* scale); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int compress_blocks(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, const uint32_t blocks) { - return compress_blocks_fallback(input, scale, output, blocks); -} +/* Clearer alias for pernix_scale_f32. */ +PERNIX_API pernix_status pernix_decompression_scale_f32(float bmax, u8 bit_width, float* scale); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return decompress_block_fallback(input, scale, output); -} +/* Clearer alias for pernix_scale_f64. */ +PERNIX_API pernix_status pernix_decompression_scale_f64(double bmax, u8 bit_width, double* scale); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_block(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return decompress_block_fallback(input, scale, output); -} +/* Computes 1 / scale for the compression API from a valid forward scale. */ +PERNIX_API pernix_status pernix_inverse_scale_f32(float scale, float* inverse_scale); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, const uint32_t blocks) { - return decompress_blocks_fallback(input, scale, output, blocks); -} +/* Computes 1 / scale for the compression API from a valid forward scale. */ +PERNIX_API pernix_status pernix_inverse_scale_f64(double scale, double* inverse_scale); -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int decompress_blocks(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, const uint32_t blocks) { - return decompress_blocks_fallback(input, scale, output, blocks); -} -#endif -} // namespace pernix +/* Computes the inverse compression scale directly from bmax and bit width. */ +PERNIX_API pernix_status pernix_compression_scale_f32(float bmax, u8 bit_width, float* inverse_scale); -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif +/* Computes the inverse compression scale directly from bmax and bit width. */ +PERNIX_API pernix_status pernix_compression_scale_f64(double bmax, u8 bit_width, double* inverse_scale); -/** - * @brief C ABI wrapper for compressing one single-precision block. - * - * @param bit_width bit width per quantized value (8 to 16). - * @param input pointer to the input block. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int compress_block(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); +PERNIX_API pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, + float inverse_scale, void* output); -/** - * @brief C ABI wrapper for compressing one double-precision block. - * - * @param bit_width bit width per quantized value (8 to 16). - * @param input pointer to the input block. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int compress_block_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); +PERNIX_API pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, + float inverse_scale, void* output, u32 blocks); -/** - * @brief C ABI wrapper for compressing multiple single-precision blocks. - * - * @param bit_width bit width per quantized value (8 to 16). - * @param input pointer to the first input value. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @param blocks number of consecutive blocks to process. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int compress_blocks(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, uint32_t blocks); +PERNIX_API pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output, bool sign_values); -/** - * @brief C ABI wrapper for compressing multiple double-precision blocks. - * - * @param bit_width bit width per quantized value (8 to 16). - * @param input pointer to the first input value. - * @param scale scaling factor used during quantization. - * @param output pointer to the destination compressed bytes. - * @param blocks number of consecutive blocks to process. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int compress_blocks_f64(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); +PERNIX_API pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output, u32 blocks, bool sign_values); -/** - * @brief C ABI wrapper for decompressing one single-precision block. - * - * @param bit_width bit width per packed value (1 to 24). - * @param input pointer to the packed input block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int decompress_block(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); +PERNIX_API pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, + double inverse_scale, void* output); -/** - * @brief C ABI wrapper for decompressing one double-precision block. - * - * @param bit_width bit width per packed value (1 to 24). - * @param input pointer to the packed input block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int decompress_block_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); +PERNIX_API pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, + double inverse_scale, void* output, u32 blocks); -/** - * @brief C ABI wrapper for decompressing multiple single-precision blocks. - * - * @param bit_width bit width per packed value (1 to 24). - * @param input pointer to the first packed block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @param blocks number of consecutive blocks to process. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int decompress_blocks(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, uint32_t blocks); +PERNIX_API pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, bool sign_values); -/** - * @brief C ABI wrapper for decompressing multiple double-precision blocks. - * - * @param bit_width bit width per packed value (1 to 24). - * @param input pointer to the first packed block. - * @param scale scaling factor used to reconstruct floating-point values. - * @param output pointer to the destination decompressed values. - * @param blocks number of consecutive blocks to process. - * @return int status code (0 for success, non-zero for invalid arguments or unsupported bit width). - */ -int decompress_blocks_f64(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); +PERNIX_API pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, u32 blocks, bool sign_values); #ifdef __cplusplus } -} // namespace pernix #endif -#endif // PERNIX_H \ No newline at end of file +#endif // PERNIX_H diff --git a/include/pernix/pernix.hpp b/include/pernix/pernix.hpp new file mode 100644 index 0000000..2358659 --- /dev/null +++ b/include/pernix/pernix.hpp @@ -0,0 +1,246 @@ +#ifndef PERNIX_HPP +#define PERNIX_HPP + +#include +#include + +#include +#include + +namespace pernix { +using Status = pernix_status; + +constexpr u8 min_bit_width() { + return detail::min_bit_width; +} + +constexpr u8 max_bit_width() { + return detail::max_bit_width; +} + +constexpr bool is_valid_bit_width(const u8 bit_width) { + return detail::is_valid_bit_width(bit_width); +} + +constexpr bool is_valid_block_size(const u32 block_size) { + return detail::is_valid_block_size(block_size); +} + +constexpr u32 compressed_block_size() { + return detail::fixed_block_size; +} + +constexpr u32 elements_per_block(const u8 bit_width) { + return detail::elements_per_block(bit_width); +} + +template +__always_inline Status scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT& scale) { + return detail::scale_from_bmax(bmax, bit_width, &scale); +} + +template +__always_inline Status decompression_scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT& scale) { + return scale_from_bmax(bmax, bit_width, scale); +} + +template +__always_inline Status inverse_scale(const FloatT scale, FloatT& inverse_scale_value) { + return detail::inverse_scale(scale, &inverse_scale_value); +} + +template +__always_inline Status compression_scale_from_bmax(const FloatT bmax, const u8 bit_width, FloatT& inverse_scale_value) { + return detail::compression_scale_from_bmax(bmax, bit_width, &inverse_scale_value); +} + +__always_inline const char* status_string(const Status status) { + return detail::status_string(status); +} + +namespace detail { +template +__always_inline Status validate_compress_spans(const u8 bit_width, const u32 block_size, const std::span input, + const std::span output, const u32 blocks) { + if (!is_valid_bit_width(bit_width)) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + if (blocks == 0) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + const usize required_input = static_cast(elements_per_block(bit_width, block_size)) * blocks; + const usize required_output = static_cast(block_size) * blocks; + return input.size() >= required_input && output.size() >= required_output ? PERNIX_STATUS_OK : PERNIX_STATUS_INVALID_ARGUMENT; +} + +template +__always_inline Status validate_decompress_spans(const u8 bit_width, const u32 block_size, const std::span input, + const std::span output, const u32 blocks) { + if (!is_valid_bit_width(bit_width)) { + return PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH; + } + if (!is_valid_block_size(block_size)) { + return PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE; + } + if (blocks == 0) { + return PERNIX_STATUS_INVALID_ARGUMENT; + } + + const usize required_input = static_cast(block_size) * blocks; + const usize required_output = static_cast(elements_per_block(bit_width, block_size)) * blocks; + return input.size() >= required_input && output.size() >= required_output ? PERNIX_STATUS_OK : PERNIX_STATUS_INVALID_ARGUMENT; +} +} // namespace detail + +PERNIX_API static Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, std::span output) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { + return status; + } + return detail::compress_block(backend, bit_width, block_size, input.data(), inverse_scale, output.data()); +} + +PERNIX_API static Status compress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, std::span output) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { + return status; + } + return detail::compress_block(backend, bit_width, block_size, input.data(), inverse_scale, output.data()); +} + +PERNIX_API static Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float scale, std::span output, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { + return status; + } + return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); +} + +PERNIX_API static Status decompress_block(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double scale, std::span output, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, 1); status != PERNIX_STATUS_OK) { + return status; + } + return detail::decompress_block(backend, bit_width, block_size, input.data(), scale, output.data(), sign_values); +} + +PERNIX_API static Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, std::span output, const u32 blocks) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { + return status; + } + return detail::compress_blocks(backend, bit_width, block_size, input.data(), inverse_scale, output.data(), blocks); +} + +PERNIX_API static Status compress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, std::span output, const u32 blocks) { + if (const Status status = detail::validate_compress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { + return status; + } + return detail::compress_blocks(backend, bit_width, block_size, input.data(), inverse_scale, output.data(), blocks); +} + +PERNIX_API static Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const float scale, std::span output, const u32 blocks, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { + return status; + } + return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, sign_values); +} + +PERNIX_API static Status decompress_blocks(Backend backend, const u8 bit_width, const u32 block_size, const std::span input, + const double scale, std::span output, const u32 blocks, const bool sign_values = true) { + if (const Status status = detail::validate_decompress_spans(bit_width, block_size, input, output, blocks); status != PERNIX_STATUS_OK) { + return status; + } + return detail::decompress_blocks(backend, bit_width, block_size, input.data(), scale, output.data(), blocks, sign_values); +} + +// convenience overloads without backend (defaults to Auto) +PERNIX_API static Status compress_block(const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, const std::span output) { + return compress_block(Backend::Auto, bit_width, block_size, input, inverse_scale, output); +} + +PERNIX_API static Status compress_block(const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, const std::span output) { + return compress_block(Backend::Auto, bit_width, block_size, input, inverse_scale, output); +} + +PERNIX_API static Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, const float scale, + const std::span output, const bool sign_values = true) { + return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); +} + +PERNIX_API static Status decompress_block(const u8 bit_width, const u32 block_size, const std::span input, const double scale, + const std::span output, const bool sign_values = true) { + return decompress_block(Backend::Auto, bit_width, block_size, input, scale, output, sign_values); +} + +PERNIX_API static Status compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const float inverse_scale, const std::span output, const u32 blocks) { + return compress_blocks(Backend::Auto, bit_width, block_size, input, inverse_scale, output, blocks); +} + +PERNIX_API static Status compress_blocks(const u8 bit_width, const u32 block_size, const std::span input, + const double inverse_scale, const std::span output, const u32 blocks) { + return compress_blocks(Backend::Auto, bit_width, block_size, input, inverse_scale, output, blocks); +} + +PERNIX_API static Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, const float scale, + const std::span output, const u32 blocks, const bool sign_values = true) { + return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); +} + +PERNIX_API static Status decompress_blocks(const u8 bit_width, const u32 block_size, const std::span input, const double scale, + const std::span output, const u32 blocks, const bool sign_values = true) { + return decompress_blocks(Backend::Auto, bit_width, block_size, input, scale, output, blocks, sign_values); +} + +// convenience overloads without backend and without block_size (defaults to 64) +PERNIX_API static Status compress_block(const u8 bit_width, const std::span input, const float inverse_scale, + const std::span output) { + return compress_block(Backend::Auto, bit_width, 64, input, inverse_scale, output); +} + +PERNIX_API static Status compress_block(const u8 bit_width, const std::span input, const double inverse_scale, + const std::span output) { + return compress_block(Backend::Auto, bit_width, 64, input, inverse_scale, output); +} + +PERNIX_API static Status decompress_block(const u8 bit_width, const std::span input, const float scale, + const std::span output, const bool sign_values = true) { + return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); +} + +PERNIX_API static Status decompress_block(const u8 bit_width, const std::span input, const double scale, + const std::span output, const bool sign_values = true) { + return decompress_block(Backend::Auto, bit_width, 64, input, scale, output, sign_values); +} + +PERNIX_API static Status compress_blocks(const u8 bit_width, const std::span input, const float inverse_scale, + const std::span output, const u32 blocks) { + return compress_blocks(Backend::Auto, bit_width, 64, input, inverse_scale, output, blocks); +} + +PERNIX_API static Status compress_blocks(const u8 bit_width, const std::span input, const double inverse_scale, + const std::span output, const u32 blocks) { + return compress_blocks(Backend::Auto, bit_width, 64, input, inverse_scale, output, blocks); +} + +PERNIX_API static Status decompress_blocks(const u8 bit_width, const std::span input, const float scale, + const std::span output, const u32 blocks, const bool sign_values = true) { + return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); +} + +PERNIX_API static Status decompress_blocks(const u8 bit_width, const std::span input, const double scale, + const std::span output, const u32 blocks, const bool sign_values = true) { + return decompress_blocks(Backend::Auto, bit_width, 64, input, scale, output, blocks, sign_values); +} +} // namespace pernix + +#endif // PERNIX_HPP diff --git a/include/pernix/simd_compat.h b/include/pernix/simd_compat.h index c95c1ee..dbc0154 100644 --- a/include/pernix/simd_compat.h +++ b/include/pernix/simd_compat.h @@ -1,63 +1,37 @@ #ifndef PERNIX_SIMD_COMPAT_H #define PERNIX_SIMD_COMPAT_H +#include + #include #include #if defined(PERNIX_USE_SIMDE) #define SIMDE_ENABLE_NATIVE_ALIASES #undef SIMDE_X86_AVX512FP16_NATIVE +#if defined(__clang__) +#define SIMDE_X86_AVX512BF16_NATIVE +#endif // #define SIMDE_NO_NATIVE +#if defined(PERNIX_BACKEND_X86) #include -#include #include - -// #ifndef __mmask8 -// typedef uint8_t __mmask8; -// #endif -// #ifndef __mmask16 -// typedef uint16_t __mmask16; -// #endif -// #ifndef __mmask32 -// typedef uint32_t __mmask32; -// #endif -// #ifndef __mmask64 -// typedef uint64_t __mmask64; -// #endif +#include +#elif defined(PERNIX_BACKEND_ARM64_NEON) +#include +#elif defined(PERNIX_BACKEND_ARM64_SVE) || defined(PERNIX_BACKEND_ARM64_SVE2) +#include +#endif #elif defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) #include +#elif defined(__aarch64__) || defined(__arm64ec__) +#ifdef __ARM_FEATURE_SVE +#include #endif - -#ifndef __always_inline -#if defined(__GNUC__) || defined(__clang__) -#define __always_inline inline __attribute__((always_inline)) -#elif defined(_MSC_VER) -#define __always_inline __forceinline -#else -#define __always_inline inline +#ifdef __ARM_NEON +#include #endif #endif -template - requires(std::is_integral_v && sizeof(T) <= 8) -static constexpr T tail_mask(const uint8_t bit_width, const uint32_t remaining_elements) { - const uint32_t tail_bits = remaining_elements * bit_width; - const uint32_t tail_bytes = (tail_bits + 7u) / 8u; - if (tail_bytes == 0u) { - return static_cast(0); - } - if (tail_bytes >= 64u) { - return static_cast(~uint64_t{0}); - } - const uint64_t mask = (uint64_t{1} << tail_bytes) - 1u; - return static_cast(mask); -} - -static constexpr uint32_t tail_bytes(const uint8_t bit_width, const uint32_t remaining_elements) { - const uint32_t tail_bits = remaining_elements * bit_width; - const uint32_t tail_bytes = (tail_bits + 7u) / 8u; - return tail_bytes; -} - #endif // PERNIX_SIMD_COMPAT_H diff --git a/include/pernix/x86/avx2/compression.h b/include/pernix/x86/avx2/avx2_compression.h similarity index 64% rename from include/pernix/x86/avx2/compression.h rename to include/pernix/x86/avx2/avx2_compression.h index 357947e..efc4a4e 100644 --- a/include/pernix/x86/avx2/compression.h +++ b/include/pernix/x86/avx2/avx2_compression.h @@ -1,10 +1,11 @@ #ifndef PERNIX_AVX2_COMPRESSION_H #define PERNIX_AVX2_COMPRESSION_H -#include -#include +#include #include +#include +#include #include #include #include @@ -12,11 +13,24 @@ namespace pernix { namespace internal { -template +template + requires(LANES <= 8) +__always_inline __m256i mm256_tail_mask_epi32() { + return _mm256_setr_epi32(LANES > 0 ? -1 : 0, LANES > 1 ? -1 : 0, LANES > 2 ? -1 : 0, LANES > 3 ? -1 : 0, LANES > 4 ? -1 : 0, + LANES > 5 ? -1 : 0, LANES > 6 ? -1 : 0, LANES > 7 ? -1 : 0); +} + +template + requires(LANES <= 4) +__always_inline __m256i mm256_tail_mask_epi64() { + return _mm256_setr_epi64x(LANES > 0 ? -1 : 0, LANES > 1 ? -1 : 0, LANES > 2 ? -1 : 0, LANES > 3 ? -1 : 0); +} + +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __always_inline __m256i mm256_clamp_signed_epi32(__m256i input) { - constexpr int32_t min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); - constexpr int32_t max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); + constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), _mm256_set1_epi32(max_value)); } @@ -27,7 +41,7 @@ __always_inline __m256i mm256_clamp_signed_epi32(__m256i input) { * @param scale per-lane scale factor. * @return __m128i quantized values. */ -__always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& scale) { +__always_inline __m128i mm_quantize_ps_epi32(__m128 input, __m128 scale) { const __m128 scaled = _mm_mul_ps(input, scale); // const __m128 rounded = _mm_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); return _mm_cvtps_epi32(scaled); @@ -40,7 +54,7 @@ __always_inline __m128i mm_quantize_ps_epi32(const __m128& input, const __m128& * @param scale per-lane scale factor. * @return __m128i quantized values in the low lanes. */ -__always_inline __m128i mm_quantize_pd_epi32(const __m128d& input, const __m128d& scale) { +__always_inline __m128i mm_quantize_pd_epi32(__m128d input, __m128d scale) { const __m128d scaled = _mm_mul_pd(input, scale); return _mm_cvtpd_epi32(scaled); } @@ -52,7 +66,7 @@ __always_inline __m128i mm_quantize_pd_epi32(const __m128d& input, const __m128d * @param scale per-lane scale factor. * @return __m256i quantized values. */ -__always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m256& scale) { +__always_inline __m256i mm256_quantize_ps_epi32(__m256 input, __m256 scale) { const __m256 scaled = _mm256_mul_ps(input, scale); // const __m256 rounded = _mm256_round_ps(scaled, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); return _mm256_cvtps_epi32(scaled); @@ -65,7 +79,7 @@ __always_inline __m256i mm256_quantize_ps_epi32(const __m256& input, const __m25 * @param scale per-lane scale factor. * @return __m128i quantized values in the low lanes. */ -__always_inline __m128i mm256_quantize_pd_epi32(const __m256d& input, const __m256d& scale) { +__always_inline __m128i mm256_quantize_pd_epi32(__m256d input, __m256d scale) { const __m256d scaled = _mm256_mul_pd(input, scale); return _mm256_cvtpd_epi32(scaled); } @@ -118,14 +132,14 @@ __always_inline static __m256i _mm256_srlv_epi16(const __m256i a, const __m256i /** * @brief Blend 8-bit lanes by expanding a scalar mask value. */ -__always_inline static __m128i mm_blend_epi8(const __m128i X, const __m128i Y, const int8_t M) { +__always_inline static __m128i mm_blend_epi8(const __m128i X, const __m128i Y, const i8 M) { return _mm_blendv_epi8(X, Y, _mm_set1_epi8(M)); } /** * @brief Blend 8-bit lanes in 256-bit vectors by expanding a scalar mask value. */ -__always_inline static __m256i mm256_blend_epi8(const __m256i X, const __m256i Y, const int8_t M) { +__always_inline static __m256i mm256_blend_epi8(const __m256i X, const __m256i Y, const i8 M) { return _mm256_blendv_epi8(X, Y, _mm256_set1_epi8(M)); } @@ -133,10 +147,10 @@ __always_inline static __m256i mm256_blend_epi8(const __m256i X, const __m256i Y * @brief Emulate per-byte left shifts on 128-bit vectors. */ __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0xff00); + const __m128i mask = _mm_set1_epi16(static_cast(0xff00u)); const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); - return mm_blend_epi8(low_half, high_half, 0xaa); + return mm_blend_epi8(low_half, high_half, static_cast(0xaau)); } /** @@ -146,17 +160,17 @@ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i coun const __m128i mask = _mm_set1_epi16(0x00ff); const __m128i low_half = _mm_srlv_epi16(_mm_and_si128(mask, a), _mm_and_si128(mask, count)); const __m128i high_half = _mm_srlv_epi16(a, _mm_srli_epi16(count, 8)); - return mm_blend_epi8(low_half, high_half, 0xaa); + return mm_blend_epi8(low_half, high_half, static_cast(0xaau)); } /** * @brief Emulate per-byte left shifts on 256-bit vectors. */ __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0xff00); + const __m256i mask = _mm256_set1_epi16(static_cast(0xff00u)); const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); - return mm256_blend_epi8(low_half, high_half, 0xaa); + return mm256_blend_epi8(low_half, high_half, static_cast(0xaau)); } /** @@ -166,37 +180,37 @@ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i c const __m256i mask = _mm256_set1_epi16(0x00ff); const __m256i low_half = _mm256_srlv_epi16(_mm256_and_si256(mask, a), _mm256_and_si256(mask, count)); const __m256i high_half = _mm256_srlv_epi16(a, _mm256_srli_epi16(count, 8)); - return mm256_blend_epi8(low_half, high_half, 0xaa); + return mm256_blend_epi8(low_half, high_half, static_cast(0xaau)); } #endif /** * @brief Pack four 32-bit values for bit widths 1 through 3. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) -__always_inline auto mm_pack_epi32_avx2_1to3(__m128i& input) -> __m128i { - constexpr uint32_t bitmask = (1U << BIT_WIDTH) - 1U; - const __m128i masked = _mm_and_si128(input, _mm_set1_epi32(static_cast(bitmask))); +__always_inline auto mm_pack_epi32_avx2_1to3(__m128i input) -> __m128i { + constexpr u32 bitmask = (1U << BIT_WIDTH) - 1U; + const __m128i masked = _mm_and_si128(input, _mm_set1_epi32(static_cast(bitmask))); - alignas(16) uint32_t lanes[4]; + alignas(16) u32 lanes[4]; _mm_storeu_si128(reinterpret_cast<__m128i*>(lanes), masked); - const uint32_t packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ((lanes[2] & bitmask) << (2 * BIT_WIDTH)) | - ((lanes[3] & bitmask) << (3 * BIT_WIDTH)); + const u32 packed = (lanes[0] & bitmask) | ((lanes[1] & bitmask) << BIT_WIDTH) | ((lanes[2] & bitmask) << (2 * BIT_WIDTH)) | + ((lanes[3] & bitmask) << (3 * BIT_WIDTH)); - return _mm_cvtsi32_si128(static_cast(packed)); + return _mm_cvtsi32_si128(static_cast(packed)); } /** * @brief Pack eight 32-bit values for bit widths 1 through 3. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 3) -__always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { - constexpr uint32_t bitmask = (1u << BIT_WIDTH) - 1u; +__always_inline __m256i mm256_pack_epi32_avx2_1to3(__m256i input) { + constexpr u32 bitmask = (1u << BIT_WIDTH) - 1u; - const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(static_cast(bitmask))); + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(static_cast(bitmask))); const __m256i shifts = _mm256_setr_epi32(0 * BIT_WIDTH, 1 * BIT_WIDTH, 2 * BIT_WIDTH, 3 * BIT_WIDTH, 4 * BIT_WIDTH, 5 * BIT_WIDTH, 6 * BIT_WIDTH, 7 * BIT_WIDTH); @@ -211,10 +225,11 @@ __always_inline __m256i mm256_pack_epi32_avx2_1to3(const __m256i& input) { return _mm256_castsi128_si256(x); } -__always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i& input) { - const __m256i zero = _mm256_setzero_si256(); +__always_inline __m256i mm256_pack_epi32_avx2_4(__m256i input) { + const __m256i zero = _mm256_setzero_si256(); + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(0x0f)); - const __m256i packed16 = _mm256_packus_epi32(input, zero); + const __m256i packed16 = _mm256_packus_epi32(masked, zero); const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); const __m256i packed8 = _mm256_packus_epi16(permuted, zero); @@ -229,12 +244,12 @@ __always_inline __m256i mm256_pack_epi32_avx2_4(const __m256i& input) { /** * @brief Pack four 32-bit values for bit widths 9 through 16. */ -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { - using tables = pack_tables_avx2_16; - constexpr uint16_t bitmask = (1 << BIT_WIDTH) - 1; - const __m128i masked = _mm_and_si128(input, _mm_set1_epi16(bitmask)); +__always_inline auto mm_pack_epi32_avx2_9to16(__m128i input) -> __m128i { + using tables = pack_tables_avx2_16; + constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; + const __m128i masked = _mm_and_si128(input, _mm_set1_epi16(bitmask)); __m128i combined; if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { @@ -262,13 +277,13 @@ __always_inline auto mm_pack_epi32_avx2_9to16(__m128i& input) -> __m128i { /** * @brief Pack eight 32-bit values for bit widths 9 through 16. */ -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i& input) -> __m256i { - using tables = pack_tables_avx2_16; - constexpr uint16_t bitmask = (1 << BIT_WIDTH) - 1; - const __m128i packed = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); - const __m128i masked = _mm_and_si128(packed, _mm_set1_epi16(bitmask)); +__always_inline auto mm256_pack_epi32_avx2_9to16(__m256i input) -> __m256i { + using tables = pack_tables_avx2_16; + constexpr u16 bitmask = (1 << BIT_WIDTH) - 1; + const __m128i packed = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); + const __m128i masked = _mm_and_si128(packed, _mm_set1_epi16(bitmask)); __m128i combined; if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15 || BIT_WIDTH == 16) { @@ -293,17 +308,18 @@ __always_inline auto mm256_pack_epi32_avx2_9to16(const __m256i& input) -> __m256 return _mm256_castsi128_si256(combined); } -template +template requires(BIT_WIDTH >= 5 && BIT_WIDTH <= 7) -__always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i { - const __m256i zero = _mm256_setzero_si256(); +__always_inline auto mm256_pack_epi32_avx2_5to7(__m256i input) -> __m256i { + const __m256i zero = _mm256_setzero_si256(); + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32((1 << BIT_WIDTH) - 1)); - const __m256i packed16 = _mm256_packus_epi32(input, zero); + const __m256i packed16 = _mm256_packus_epi32(masked, zero); const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); const __m256i packed8 = _mm256_packus_epi16(permuted, zero); const __m256i even = _mm256_and_si256(packed8, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(packed8, _mm256_set1_epi16(0xFF00)); + const __m256i odd = _mm256_and_si256(packed8, _mm256_set1_epi16(static_cast(0xFF00u))); const __m256i pair16 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); const __m256i extended = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(pair16)); @@ -314,12 +330,12 @@ __always_inline auto mm256_pack_epi32_avx2_5to7(const __m256i& input) -> __m256i /** * @brief Pack eight 32-bit values for bit widths 17 through 24. */ -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m256i { - using tables = pack_tables_avx2_24; - constexpr uint32_t bitmask = (1 << BIT_WIDTH) - 1; - const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(bitmask)); +__always_inline auto mm256_pack_epi32_avx2_17to24(__m256i input) -> __m256i { + using tables = pack_tables_avx2_24; + constexpr u32 bitmask = (1 << BIT_WIDTH) - 1; + const __m256i masked = _mm256_and_si256(input, _mm256_set1_epi32(bitmask)); __m256i combined; if constexpr (BIT_WIDTH == 24) { @@ -348,9 +364,9 @@ __always_inline auto mm256_pack_epi32_avx2_17to24(const __m256i& input) -> __m25 /** * @brief Pack aligned 8-bit or 16-bit values from four 32-bit lanes. */ -template +template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { +auto mm_pack_aligned_epi32_avx2(__m128i input) -> __m128i { if constexpr (BIT_WIDTH == 8) { return _mm_packus_epi16(_mm_packs_epi32(input, _mm_setzero_si128()), _mm_setzero_si128()); } else { @@ -361,9 +377,9 @@ auto mm_pack_aligned_epi32_avx2(__m128i& input) -> __m128i { /** * @brief Dispatch to the appropriate 128-bit AVX2 packer for the selected bit width. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 16) -auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { +auto mm_pack_epi32_avx2(__m128i input) -> __m128i { if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 3) { return internal::mm_pack_epi32_avx2_1to3(input); } else if constexpr (BIT_WIDTH >= 4 && BIT_WIDTH <= 8) { @@ -379,9 +395,9 @@ auto mm_pack_epi32_avx2(__m128i& input) -> __m128i { /** * @brief Pack aligned 8-bit or 16-bit values from eight 32-bit lanes. */ -template +template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -__m256i mm256_pack_aligned_epi32_avx2(const __m256i& input) { +__m256i mm256_pack_aligned_epi32_avx2(__m256i input) { if constexpr (BIT_WIDTH == 8) { const __m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(input), _mm256_extracti128_si256(input, 1)); const __m128i packed8 = _mm_packs_epi16(packed16, _mm_setzero_si128()); @@ -394,9 +410,9 @@ __m256i mm256_pack_aligned_epi32_avx2(const __m256i& input) { /** * @brief Dispatch to the appropriate 256-bit AVX2 packer for the selected bit width. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__m256i mm256_pack_epi32_avx2(const __m256i& input) { +__m256i mm256_pack_epi32_avx2(__m256i input) { if constexpr (BIT_WIDTH == 8 || BIT_WIDTH == 16) { return internal::mm256_pack_aligned_epi32_avx2(input); } else { @@ -414,7 +430,7 @@ __m256i mm256_pack_epi32_avx2(const __m256i& input) { } return _mm256_setzero_si256(); } -} // namespace internal +} // namespace internal /** * @brief Compress a single block of float using AVX2 instructions. @@ -429,37 +445,38 @@ __m256i mm256_pack_epi32_avx2(const __m256i& input) { * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_avx2(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; +int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; std::memset(output, 0, BLOCK_SIZE); const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 8 - for (uint32_t iter = 0; iter < iterations_8; iter++) { + for (u32 iter = 0; iter < iterations_8; iter++) { const __m256 source = _mm256_loadu_ps(input); const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); - const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); std::memcpy(output, &packed, BIT_WIDTH); - input += 8; + input += 8; output += BIT_WIDTH; } if constexpr (remaining) { - std::vector block_values(remaining); -#pragma GCC unroll 8 - for (uint32_t i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], scale))); - } - - internal::pack_epi32_fallback(block_values, output); + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + const __m256 source = _mm256_maskload_ps(input, internal::mm256_tail_mask_epi32()); + const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + std::memcpy(output, &packed, tail_bytes); } return 0; @@ -478,40 +495,54 @@ int mm256_compress_block_avx2(const float_t* __restrict__ input, const float_t s * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_avx2(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; +int mm256_compress_block_avx2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; std::memset(output, 0, BLOCK_SIZE); const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 8 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256d source1 = _mm256_loadu_pd(input); - const __m256d source2 = _mm256_loadu_pd(input + 4); - const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); - const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); - __m256i combined = _mm256_castsi128_si256(quantized1); - combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_avx2(internal::mm256_clamp_signed_epi32(combined)); - // _mm_storeu_si128(reinterpret_cast<__m128i*>(output), _mm256_castsi256_si128(packed)); + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256d source1 = _mm256_loadu_pd(input); + const __m256d source2 = _mm256_loadu_pd(input + 4); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(combined); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); std::memcpy(output, &packed, BIT_WIDTH); - input += 8; + input += 8; output += BIT_WIDTH; } if constexpr (remaining) { - std::vector block_values(remaining); -#pragma GCC unroll 8 - for (uint32_t i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], scale))); - } - - internal::pack_epi32_fallback(block_values, output); + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + constexpr u8 first_lanes = remaining < 4 ? remaining : 4; + constexpr u8 second_lanes = remaining > 4 ? remaining - 4 : 0; + + const __m256d source1 = _mm256_maskload_pd(input, internal::mm256_tail_mask_epi64()); + const __m256d source2 = [&] { + if constexpr (second_lanes > 0) { + return _mm256_maskload_pd(input + 4, internal::mm256_tail_mask_epi64()); + } else { + return _mm256_setzero_pd(); + } + }(); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(combined); + const __m256i packed = internal::mm256_pack_epi32_avx2(packed_input); + std::memcpy(output, &packed, tail_bytes); } return 0; @@ -531,16 +562,18 @@ int mm256_compress_block_avx2(const double_t* __restrict__ input, const double_t * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_avx2(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - const float_t* block_input = input; - uint8_t* block_output = output; +int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f32* block_input = input; + u8* block_output = output; - for (uint32_t block = 0; block < blocks; block++) { + for (u32 block = 0; block < blocks; block++) { mm256_compress_block_avx2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } @@ -561,87 +594,23 @@ int mm256_compress_blocks_avx2(const float_t* __restrict__ input, const float_t * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_avx2(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - const double_t* block_input = input; - uint8_t* block_output = output; +int mm256_compress_blocks_avx2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - for (uint32_t block = 0; block < blocks; block++) { + const f64* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; block++) { mm256_compress_block_avx2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } return 0; } -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Compress a single 512-bit block using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_compress_block_avx2(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single 512-bit block using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_compress_block_f64_avx2(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress multiple 512-bit blocks using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_blocks_avx2(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Compress multiple 512-bit blocks using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_blocks_f64_avx2(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif +} // namespace pernix #endif // PERNIX_AVX2_COMPRESSION_H diff --git a/include/pernix/x86/avx2/decompression.h b/include/pernix/x86/avx2/avx2_decompression.h similarity index 57% rename from include/pernix/x86/avx2/decompression.h rename to include/pernix/x86/avx2/avx2_decompression.h index 93cbce5..027b71f 100644 --- a/include/pernix/x86/avx2/decompression.h +++ b/include/pernix/x86/avx2/avx2_decompression.h @@ -1,9 +1,9 @@ #ifndef PERNIX_AVX2_DECOMPRESSION_H #define PERNIX_AVX2_DECOMPRESSION_H -#include -#include +#include #include +#include #include #include @@ -30,7 +30,7 @@ __always_inline __m256i mm256_convert_vmask_epi64(const __mmask8 mask8) { /** * @brief Dequantize four 32-bit integers to floats. */ -__always_inline __m128 mm_dequantize_epi32(const __m128i& input, const __m128& scale) { +__always_inline __m128 mm_dequantize_epi32(__m128i input, __m128 scale) { const __m128 converted = _mm_cvtepi32_ps(input); return _mm_mul_ps(converted, scale); } @@ -39,16 +39,16 @@ __always_inline __m128 mm_dequantize_epi32(const __m128i& input, const __m128& s __always_inline __m128d convert_epi64_pd(const __m128i v) { __m128i xH = _mm_srai_epi32(v, 16); xH = _mm_blend_epi16(xH, _mm_setzero_si128(), 0x33); - xH = _mm_add_epi64(xH, _mm_castpd_si128(_mm_set1_pd(442721857769029238784.))); // 3*2^67 - const __m128i xL = _mm_blend_epi16(v, _mm_castpd_si128(_mm_set1_pd(0x0010000000000000)), 0x88); // 2^52 - const __m128d f = _mm_sub_pd(_mm_castsi128_pd(xH), _mm_set1_pd(442726361368656609280.)); // 3*2^67 + 2^52 + xH = _mm_add_epi64(xH, _mm_castpd_si128(_mm_set1_pd(442721857769029238784.))); // 3*2^67 + const __m128i xL = _mm_blend_epi16(v, _mm_castpd_si128(_mm_set1_pd(0x0010000000000000)), 0x88); // 2^52 + const __m128d f = _mm_sub_pd(_mm_castsi128_pd(xH), _mm_set1_pd(442726361368656609280.)); // 3*2^67 + 2^52 return _mm_add_pd(f, _mm_castsi128_pd(xL)); } /** * @brief Dequantize two 64-bit integers to doubles. */ -__always_inline __m128d mm_dequantize_epi64_pd(const __m128i& input, const __m128d& scale) { +__always_inline __m128d mm_dequantize_epi64_pd(__m128i input, __m128d scale) { const __m128d converted = convert_epi64_pd(input); return _mm_mul_pd(converted, scale); } @@ -56,7 +56,7 @@ __always_inline __m128d mm_dequantize_epi64_pd(const __m128i& input, const __m12 /** * @brief Dequantize eight 32-bit integers to floats. */ -__always_inline __m256 mm256_dequantize_epi32(const __m256i& input, const __m256& scale) { +__always_inline __m256 mm256_dequantize_epi32(__m256i input, __m256 scale) { const __m256 converted = _mm256_cvtepi32_ps(input); return _mm256_mul_ps(converted, scale); } @@ -74,7 +74,7 @@ __always_inline __m256d convert_epi64_pd(__m256i v) { /** * @brief Dequantize four 64-bit integers to doubles. */ -__always_inline __m256d mm256_dequantize_epi64_pd(const __m256i& input, const __m256d& scale) { +__always_inline __m256d mm256_dequantize_epi64_pd(__m256i input, __m256d scale) { const __m256d converted = convert_epi64_pd(input); return _mm256_mul_pd(converted, scale); } @@ -82,9 +82,9 @@ __always_inline __m256d mm256_dequantize_epi64_pd(const __m256i& input, const __ /** * @brief Unpack four aligned 8-bit or 16-bit values directly from the input buffer. */ -template +template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -__m128i mm_unpack_aligned_epi32_avx2(const uint8_t* __restrict__ input) { +__m128i mm_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { if constexpr (BIT_WIDTH == 8) { const __m128i source = _mm_loadu_si32(input); if constexpr (SIGN_VALUES) { @@ -105,19 +105,19 @@ __m128i mm_unpack_aligned_epi32_avx2(const uint8_t* __restrict__ input) { /** * @brief Unpack four values using the table-driven AVX2 shuffle path. */ -template +template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -__m128i mm_unpack_epi32_avx2(const uint8_t* __restrict__ input) { - using unpack_table = unpack_tables_avx2; - constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; +__m128i mm_unpack_epi32_avx2(const u8* __restrict__ input) { + using unpack_table = unpack_tables_avx2; + constexpr usize packed_bytes = (4 * BIT_WIDTH + 7) / 8; __m128i source = _mm_setzero_si128(); std::memcpy(&source, input, packed_bytes); const __m128i shuffled = _mm_shuffle_epi8(source, unpack_table::get_shuffle()); - constexpr uint16_t shift = 32 - BIT_WIDTH; - __m128i shifted = _mm_sllv_epi32(shuffled, unpack_table::get_shift()); + constexpr u16 shift = 32 - BIT_WIDTH; + __m128i shifted = _mm_sllv_epi32(shuffled, unpack_table::get_shift()); if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { shifted = _mm_srai_epi32(shifted, shift); } else { @@ -130,9 +130,9 @@ __m128i mm_unpack_epi32_avx2(const uint8_t* __restrict__ input) { /** * @brief Unpack eight aligned 8-bit or 16-bit values directly from the input buffer. */ -template +template requires(BIT_WIDTH == 8 || BIT_WIDTH == 16) -__m256i mm256_unpack_aligned_epi32_avx2(const uint8_t* __restrict__ input) { +__m256i mm256_unpack_aligned_epi32_avx2(const u8* __restrict__ input) { if constexpr (BIT_WIDTH == 8) { const __m128i source = _mm_loadu_si64(input); if constexpr (SIGN_VALUES) { @@ -153,20 +153,15 @@ __m256i mm256_unpack_aligned_epi32_avx2(const uint8_t* __restrict__ input) { /** * @brief Unpack eight values using the table-driven AVX2 shuffle path. */ -template +template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -__m256i mm256_unpack_epi32_avx2(const uint8_t* __restrict__ input) { - using unpack_table = unpack_tables_avx2; - constexpr std::size_t packed_bytes = BIT_WIDTH; - - __m256i source = _mm256_setzero_si256(); - std::memcpy(&source, input, packed_bytes); - +__m256i mm256_unpack_epi32_avx2(const __m256i source) { + using unpack_table = unpack_tables_avx2; const __m256i permuted = _mm256_permutevar8x32_epi32(source, unpack_table::get_permute()); const __m256i shuffled = _mm256_shuffle_epi8(permuted, unpack_table::get_shuffle()); - constexpr uint16_t shift = 32 - BIT_WIDTH; - __m256i shifted = _mm256_sllv_epi32(shuffled, unpack_table::get_shift()); + constexpr u16 shift = 32 - BIT_WIDTH; + __m256i shifted = _mm256_sllv_epi32(shuffled, unpack_table::get_shift()); if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { shifted = _mm256_srai_epi32(shifted, shift); } else { @@ -175,7 +170,18 @@ __m256i mm256_unpack_epi32_avx2(const uint8_t* __restrict__ input) { return shifted; } -} // namespace internal + +/** + * @brief Load and unpack eight values using the table-driven AVX2 shuffle path. + */ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +__m256i mm256_unpack_epi32_avx2(const u8* __restrict__ input) { + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, BIT_WIDTH); + return mm256_unpack_epi32_avx2(source); +} +} // namespace internal /** * @brief Decompress a single block to float using AVX2 instructions. @@ -190,28 +196,34 @@ __m256i mm256_unpack_epi32_avx2(const uint8_t* __restrict__ input) { * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_avx2(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; +int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { + for (u32 iter = 0; iter < iterations_8; iter++) { const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); _mm256_storeu_ps(output, dequantized); - input += BIT_WIDTH; + input += BIT_WIDTH; output += 8; } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (uint32_t i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi32(tail_values[i], scale); - } + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, tail_bytes); + + const __m256i unpacked = internal::mm256_unpack_epi32_avx2(source); + const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); + std::memcpy(output, &dequantized, remaining * sizeof(f32)); } return 0; @@ -230,15 +242,18 @@ int mm256_decompress_block_avx2(const uint8_t* __restrict__ input, const float_t * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_avx2(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; - const __m256d scale_v = _mm256_set1_pd(scale); +int mm256_decompress_block_avx2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { + for (u32 iter = 0; iter < iterations_8; iter++) { const __m256i unpacked = internal::mm256_unpack_epi32_avx2(input); const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); @@ -249,14 +264,26 @@ int mm256_decompress_block_avx2(const uint8_t* __restrict__ input, const double_ _mm256_storeu_pd(output, dequantized1); _mm256_storeu_pd(output + 4, dequantized2); - input += BIT_WIDTH; + input += BIT_WIDTH; output += 8; } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (uint32_t i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi64(tail_values[i], scale); + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, tail_bytes); + + const __m256i unpacked = internal::mm256_unpack_epi32_avx2(source); + const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); + const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); + + const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); + const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); + + constexpr u32 first_elements = remaining < 4 ? remaining : 4; + std::memcpy(output, &dequantized1, first_elements * sizeof(f64)); + if constexpr (remaining > 4) { + std::memcpy(output + 4, &dequantized2, (remaining - 4) * sizeof(f64)); } } return 0; @@ -276,16 +303,18 @@ int mm256_decompress_block_avx2(const uint8_t* __restrict__ input, const double_ * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_avx2(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; +int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const u8* block_input = input; + f32* block_output = output; - for (uint32_t block = 0; block < blocks; block++) { + for (u32 block = 0; block < blocks; block++) { mm256_decompress_block_avx2(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -306,87 +335,23 @@ int mm256_decompress_blocks_avx2(const uint8_t* __restrict__ input, const float_ * * @note This function requires AVX2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_avx2(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; +int mm256_decompress_blocks_avx2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - for (uint32_t block = 0; block < blocks; block++) { + const u8* block_input = input; + f64* block_output = output; + + for (u32 block = 0; block < blocks; block++) { mm256_decompress_block_avx2(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Decompress a single 512-bit block to float using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_decompress_block_avx2(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single 512-bit block to double using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_decompress_block_f64_avx2(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -/** - * @brief Decompress multiple 512-bit blocks using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_decompress_blocks_avx2(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Decompress multiple 512-bit blocks to double using AVX2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 support. - */ -int mm256_decompress_blocks_f64_avx2(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif +} // namespace pernix #endif // PERNIX_AVX2_DECOMPRESSION_H diff --git a/include/pernix/x86/avx2/tables.h b/include/pernix/x86/avx2/avx2_tables.h similarity index 79% rename from include/pernix/x86/avx2/tables.h rename to include/pernix/x86/avx2/avx2_tables.h index f4f374b..e8e5368 100644 --- a/include/pernix/x86/avx2/tables.h +++ b/include/pernix/x86/avx2/avx2_tables.h @@ -7,13 +7,13 @@ #include namespace pernix::internal { -template <__uint8_t BIT_WIDTH, typename T> +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && (std::is_same_v || std::is_same_v)) struct pack_tables_avx2_16 { - alignas(64) inline static constexpr std::array permute1 = [] { + alignas(64) inline static constexpr std::array permute1 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 0, 1, 4, 5, 8, 9, 12, 13, -1, -1, -1, -1, @@ -25,7 +25,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 0, 1, 4, 5, -1, -1, 10, 11, 14, 15, -1, -1, @@ -37,7 +37,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 0, 1, 4, 5, 6, 7, 10, 11, 12, 13, -1, -1, @@ -49,7 +49,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 12) { - return std::array{ + return std::array{ 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, @@ -61,7 +61,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 0, 1, -1, -1, 6, 7, 8, 9, 10, 11, -1, -1, @@ -73,7 +73,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 14) { - return std::array{ + return std::array{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, @@ -85,7 +85,7 @@ struct pack_tables_avx2_16 { 14, 15, -1, -1 }; } else if constexpr (BIT_WIDTH == 15) { - return std::array{ + return std::array{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, @@ -97,14 +97,14 @@ struct pack_tables_avx2_16 { 14, 15, -1, -1 }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array permute2 = [] { + alignas(64) inline static constexpr std::array permute2 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 2, 3, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, @@ -116,7 +116,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 2, 3, 6, 7, 8, 9, 12, 13, -1, -1, -1, -1, @@ -128,7 +128,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 2, 3, -1, -1, 8, 9, -1, -1, 14, 15, -1, -1, @@ -140,7 +140,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 12) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, @@ -152,7 +152,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 2, 3, 4, 5, -1, -1, -1, -1, 12, 13, 14, 15, @@ -164,7 +164,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 14) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, @@ -176,7 +176,7 @@ struct pack_tables_avx2_16 { 12, 13, -1, -1 }; } else if constexpr (BIT_WIDTH == 15) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, @@ -188,14 +188,14 @@ struct pack_tables_avx2_16 { 12, 13, 14, 15 }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array permute3 = [] { + alignas(64) inline static constexpr std::array permute3 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ -1, -1, 2, 3, 6, 7, 10, 11, 14, 15, -1, -1, @@ -207,7 +207,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ -1, -1, 2, 3, 6, 7, 8, 9, 12, 13, -1, -1, @@ -219,7 +219,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ -1, -1, 2, 3, 4, 5, 8, 9, 10, 11, 14, 15, @@ -231,7 +231,7 @@ struct pack_tables_avx2_16 { -1, -1, -1, -1 }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, @@ -243,123 +243,124 @@ struct pack_tables_avx2_16 { 14, 15, -1, -1 }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array shift1 = [] { + alignas(64) inline static constexpr std::array shift1 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 0, 2, 4, 6, 0, 1, 3, 5, 0, 2, 4, 6, 0, 1, 3, 5 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 0, 4, 0, 2, 6, 0, 4, 0, 0, 4, 0, 2, 6, 0, 4, 0 }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 0, 6, 1, 7, 2, 0, 3, 0, 0, 6, 1, 7, 2, 0, 3, 0 }; } else if constexpr (BIT_WIDTH == 12) { - return std::array{ + return std::array{ 12, 8, 4, 12, 8, 4, 12, 8, 12, 8, 4, 12, 8, 4, 12, 8 }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 0, 0, 7, 4, 1, 0, 0, 5, 0, 0, 7, 4, 1, 0, 0, 5 }; } else if constexpr (BIT_WIDTH == 14) { - return std::array{ + return std::array{ 14, 12, 10, 8, 6, 4, 2, 14, 14, 12, 10, 8, 6, 4, 2, 14 }; } else if constexpr (BIT_WIDTH == 15) { - return std::array{ + return std::array{ 15, 14, 13, 12, 11, 10, 9, 8, 15, 14, 13, 12, 11, 10, 9, 8 }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array shift2 = [] { + alignas(64) inline static constexpr std::array shift2 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 9, 11, 13, 15, 8, 10, 12, 14, 9, 11, 13, 15, 8, 10, 12, 14, }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 10, 14, 8, 12, 0, 10, 14, 8, 10, 14, 8, 12, 0, 10, 14, 8, }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 11, 8, 12, 8, 13, 8, 14, 9, 11, 8, 12, 8, 13, 8, 14, 9, }; } else if constexpr (BIT_WIDTH == 12) { - return std::array{ + return std::array{ 0, 4, 8, 0, 4, 8, 0, 4, 0, 4, 8, 0, 4, 8, 0, 4, }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 13, 10, 0, 0, 14, 11, 8, 0, 13, 10, 0, 0, 14, 11, 8, 0, }; } else if constexpr (BIT_WIDTH == 14) { - return std::array{ + return std::array{ 0, 2, 4, 6, 8, 10, 12, 0, 0, 2, 4, 6, 8, 10, 12, 0, }; } else if constexpr (BIT_WIDTH == 15) { - return std::array{ + return std::array{ 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array shift3 = [] { + alignas(64) inline static constexpr std::array shift3 = [] { // clang-format off if constexpr (BIT_WIDTH == 9) { - return std::array{ + return std::array{ 0, 7, 5, 3, 1, 8, 6, 4, 0, 7, 5, 3, 1, 8, 6, 4 }; } else if constexpr (BIT_WIDTH == 10) { - return std::array{ + return std::array{ 0, 6, 2, 8, 4, 0, 6, 2, 0, 6, 2, 8, 4, 0, 6, 2, }; } else if constexpr (BIT_WIDTH == 11) { - return std::array{ + return std::array{ 0, 5, 10, 4, 9, 3, 8, 2, 0, 5, 10, 4, 9, 3, 8, 2, }; } else if constexpr (BIT_WIDTH == 13) { - return std::array{ + return std::array{ 0, 3, 6, 9, 12, 2, 5, 8, 0, 3, 6, 9, 12, 2, 5, 8, }; } - return std::array{}; + return std::array{}; // clang-format on }(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" + __always_inline static T get_permute1() { if constexpr (std::is_same_v) { return _mm256_load_si256(reinterpret_cast(permute1.data())); @@ -413,13 +414,14 @@ struct pack_tables_avx2_16 { } return T{}; } + #pragma GCC diagnostic pop }; -template <__uint8_t BIT_WIDTH, typename T> +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) struct pack_tables_avx2_24 { - alignas(64) inline static constexpr std::array permute1 = [] { + alignas(64) inline static constexpr std::array permute1 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -454,11 +456,11 @@ struct pack_tables_avx2_24 { 1, 2, 3, 5, 6, 7, -1, -1, }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array permute2 = [] { + alignas(64) inline static constexpr std::array permute2 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -493,11 +495,11 @@ struct pack_tables_avx2_24 { 0, 1, 2, 4, 5, 6, -1, -1, }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array permute3 = [] { + alignas(64) inline static constexpr std::array permute3 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { return std::array{ @@ -528,125 +530,126 @@ struct pack_tables_avx2_24 { 0, 1, 2, 4, 5, 6, 0, 0 }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array shift1 = [] { + alignas(64) inline static constexpr std::array shift1 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { - return std::array{ + return std::array{ 0, 2, 4, 6, 8, 10, 12, 14 }; } else if constexpr (BIT_WIDTH == 18) { - return std::array{ + return std::array{ 0, 4, 8, 12, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 19) { - return std::array{ + return std::array{ 0, 6, 12, 18, 5, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 20) { - return std::array{ + return std::array{ 0, 8, 16, 4, 12, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 21) { - return std::array{ + return std::array{ 0, 10, 20, 9, 19, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 22) { - return std::array{ + return std::array{ 0, 12, 2, 14, 4, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 23) { - return std::array{ + return std::array{ 0, 14, 5, 19, 10, 1, 32, 32 }; } else if constexpr (BIT_WIDTH == 24) { - return std::array{ + return std::array{ 24, 16, 8, 24, 16, 8, 24, 16 }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array shift2 = [] { + alignas(64) inline static constexpr std::array shift2 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { - return std::array{ + return std::array{ 17, 19, 21, 23, 25, 27, 29, 31 }; } else if constexpr (BIT_WIDTH == 18) { - return std::array{ + return std::array{ 18, 22, 26, 30, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 19) { - return std::array{ + return std::array{ 19, 25, 31, 32, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 20) { - return std::array{ + return std::array{ 20, 28, 32, 24, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 21) { - return std::array{ + return std::array{ 21, 31, 32, 30, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 22) { - return std::array{ + return std::array{ 22, 32, 24, 32, 26, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 23) { - return std::array{ + return std::array{ 23, 32, 28, 32, 32, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 24) { - return std::array{ + return std::array{ 0, 8, 16, 0, 8, 16, 0, 8, }; } - return std::array{}; + return std::array{}; // clang-format on }(); - alignas(64) inline static constexpr std::array shift3 = [] { + alignas(64) inline static constexpr std::array shift3 = [] { // clang-format off if constexpr (BIT_WIDTH == 17) { - return std::array{ + return std::array{ 0, 15, 13, 11, 9, 7, 5, 3 }; } else if constexpr (BIT_WIDTH == 18) { - return std::array{ + return std::array{ 32, 14, 10, 6, 2, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 19) { - return std::array{ + return std::array{ 32, 13, 7, 1, 14, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 20) { - return std::array{ + return std::array{ 32, 12, 4, 16, 8, 32, 32, 32 }; } else if constexpr (BIT_WIDTH == 21) { - return std::array{ + return std::array{ 32, 11, 1, 12, 2, 13, 32, 32 }; } else if constexpr (BIT_WIDTH == 22) { - return std::array{ + return std::array{ 32, 10, 20, 8, 18, 6, 32, 32 }; } else if constexpr (BIT_WIDTH == 23) { - return std::array{ + return std::array{ 32, 9, 18, 4, 13, 22, 32, 32 }; } - return std::array{}; + return std::array{}; // clang-format on }(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" + __always_inline static T get_permute1() { if constexpr (std::is_same_v) { return _mm256_load_si256(reinterpret_cast(permute1.data())); @@ -700,13 +703,14 @@ struct pack_tables_avx2_24 { } return T{}; } + #pragma GCC diagnostic pop }; -template +template requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24 && (std::is_same_v || std::is_same_v)) struct unpack_tables_avx2 { - alignas(32) inline static constexpr std::array permute = [] { + alignas(32) inline static constexpr std::array permute = [] { // clang-format off if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return std::array{0, -1, -1, -1, 0, 1, -1, -1}; @@ -718,26 +722,26 @@ struct unpack_tables_avx2 { // clang-format on }(); - alignas(32) inline static constexpr std::array shuffle = [] { - std::array shuffles{}; + alignas(32) inline static constexpr std::array shuffle = [] { + std::array shuffles{}; shuffles.fill(-1); - constexpr std::size_t rebase_second_half = 4 * ((BIT_WIDTH - 1) / 8); + constexpr usize rebase_second_half = 4 * ((BIT_WIDTH - 1) / 8); - for (std::size_t lane = 0; lane < 2; ++lane) { - for (std::size_t i = 0; i < 4; ++i) { - const std::size_t value_index = lane * 4 + i; + for (usize lane = 0; lane < 2; ++lane) { + for (usize i = 0; i < 4; ++i) { + const usize value_index = lane * 4 + i; - const std::size_t bit_start = value_index * BIT_WIDTH; - const std::size_t byte_start = bit_start / 8; - const std::size_t bit_offset = bit_start % 8; - const std::size_t byte_count = (bit_offset + BIT_WIDTH + 7) / 8; + const usize bit_start = value_index * BIT_WIDTH; + const usize byte_start = bit_start / 8; + const usize bit_offset = bit_start % 8; + const usize byte_count = (bit_offset + BIT_WIDTH + 7) / 8; - const std::size_t rebase = (lane == 0) ? 0 : rebase_second_half; - const std::size_t rel_byte_start = byte_start - rebase; + const usize rebase = (lane == 0) ? 0 : rebase_second_half; + const usize rel_byte_start = byte_start - rebase; - const std::size_t dst = (lane * 4 + i) * 4; - for (std::size_t k = 0; k < byte_count; ++k) { - shuffles[dst + k] = static_cast(rel_byte_start + k); + const usize dst = (lane * 4 + i) * 4; + for (usize k = 0; k < byte_count; ++k) { + shuffles[dst + k] = static_cast(rel_byte_start + k); } } } @@ -745,11 +749,11 @@ struct unpack_tables_avx2 { return shuffles; }(); - alignas(64) inline static constexpr std::array shift = [] { - std::array shifts{}; + alignas(64) inline static constexpr std::array shift = [] { + std::array shifts{}; - for (std::size_t lane = 0; lane < 8; ++lane) { - const int bit_offset = lane * BIT_WIDTH; + for (usize lane = 0; lane < 8; ++lane) { + const int bit_offset = static_cast(lane * BIT_WIDTH); const int bit_in_byte = bit_offset % 8; const int left_shift = 32 - BIT_WIDTH - bit_in_byte; shifts[lane] = left_shift; @@ -760,6 +764,7 @@ struct unpack_tables_avx2 { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" + __always_inline static __m256i get_permute() { return _mm256_load_si256(reinterpret_cast(permute.data())); } __always_inline static T get_shuffle() { @@ -777,9 +782,9 @@ struct unpack_tables_avx2 { return _mm256_load_si256(reinterpret_cast(shift.data())); } } + #pragma GCC diagnostic pop }; - } // namespace pernix::internal #endif // PERNIX_AVX2_TABLES_H diff --git a/include/pernix/x86/avx512vbmi/compression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h similarity index 70% rename from include/pernix/x86/avx512vbmi/compression.h rename to include/pernix/x86/avx512vbmi/avx512vbmi_compression.h index bc5a375..1f35579 100644 --- a/include/pernix/x86/avx512vbmi/compression.h +++ b/include/pernix/x86/avx512vbmi/avx512vbmi_compression.h @@ -1,36 +1,39 @@ #ifndef PERNIX_AVX512VBMI_COMPRESSION_H #define PERNIX_AVX512VBMI_COMPRESSION_H -#include -#include -#include #include +#include +#include +#include +#include #include +using namespace pernix::x86::internal; + namespace pernix { namespace internal { -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) static __always_inline __m512i mm512_clamp_signed_epi32(__m512i input) { - constexpr int32_t min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); - constexpr int32_t max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); + constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); return _mm512_min_epi32(_mm512_max_epi32(input, _mm512_set1_epi32(min_value)), _mm512_set1_epi32(max_value)); } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) static __always_inline __m256i mm256_clamp_signed_epi32_avx512(__m256i input) { - constexpr int32_t min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); - constexpr int32_t max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); + constexpr i32 min_value = BIT_WIDTH == 1 ? 0 : -(1 << (BIT_WIDTH - 1)); + constexpr i32 max_value = BIT_WIDTH == 1 ? 1 : ((1 << (BIT_WIDTH - 1)) - 1); return _mm256_min_epi32(_mm256_max_epi32(input, _mm256_set1_epi32(min_value)), _mm256_set1_epi32(max_value)); } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) static __always_inline __m512i mm512_clamp_signed_epi64(__m512i input) { - constexpr int64_t min_value = BIT_WIDTH == 1 ? 0 : -(int64_t{1} << (BIT_WIDTH - 1)); - constexpr int64_t max_value = BIT_WIDTH == 1 ? 1 : ((int64_t{1} << (BIT_WIDTH - 1)) - 1); + constexpr i64 min_value = BIT_WIDTH == 1 ? 0 : -(i64{1} << (BIT_WIDTH - 1)); + constexpr i64 max_value = BIT_WIDTH == 1 ? 1 : ((i64{1} << (BIT_WIDTH - 1)) - 1); return _mm512_min_epi64(_mm512_max_epi64(input, _mm512_set1_epi64(min_value)), _mm512_set1_epi64(max_value)); } @@ -38,17 +41,17 @@ static __always_inline __m512i mm512_clamp_signed_epi64(__m512i input) { * @brief Quantize sixteen float values to 32-bit integers. */ -static __always_inline __m512i mm512_quantize_ps_epi32(const __m512& input, const __m512& scale) { +static __always_inline __m512i mm512_quantize_ps_epi32(__m512 input, __m512 scale) { const __m512 scaled = _mm512_mul_ps(input, scale); return _mm512_cvtps_epi32(scaled); } -static __always_inline __m512i mm512_quantize_pd_epi64(const __m512d& input, const __m512d& scale) { +static __always_inline __m512i mm512_quantize_pd_epi64(__m512d input, __m512d scale) { const __m512d scaled = _mm512_mul_pd(input, scale); return _mm512_cvtpd_epi64(scaled); } -static __always_inline __m256i mm512_quantize_pd_epi32(const __m512d& input, const __m512d& scale) { +static __always_inline __m256i mm512_quantize_pd_epi32(__m512d input, __m512d scale) { const __m512d scaled = _mm512_mul_pd(input, scale); return _mm512_cvtpd_epi32(scaled); } @@ -96,22 +99,21 @@ static __always_inline __m256i make_m256i_from_4x64(const __m128i a, const __m12 return x; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_compress_block_avx512vbmi_1to8(const f32* __restrict__ input, const f32 scale, u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_64 = elements_per_block / 64; - constexpr uint32_t iterations_32 = (elements_per_block % 64) / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + constexpr u32 iterations_64 = elements_per_block / 64; + constexpr u32 iterations_32 = (elements_per_block % 64) / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; const __m512 scale_v = _mm512_set1_ps(scale); if constexpr (iterations_64 > 0) { #pragma GCC unroll 8 - for (uint32_t iter = 0; iter < iterations_64; ++iter) { + for (u32 iter = 0; iter < iterations_64; ++iter) { const __m512 source1 = _mm512_loadu_ps(input); const __m512 source2 = _mm512_loadu_ps(input + 16); const __m512 source3 = _mm512_loadu_ps(input + 32); @@ -132,7 +134,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - input += 64; + input += 64; output += 8 * BIT_WIDTH; } } @@ -150,7 +152,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri const __m256i packed = m256::mm256_pack_epi8_avx512vbmi_1to8(make_m256i_from_2x128(converted1, converted2)); mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } @@ -162,7 +164,7 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri const __m128i packed = m128::mm_pack_epi8_avx512vbmi_1to8(converted); mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -178,23 +180,22 @@ __always_inline int mm512_compress_block_avx512vbmi_1to8(const float_t* __restri return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_9to16(const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_compress_block_avx512vbmi_9to16(const f32* __restrict__ input, const f32 scale, u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_32 = elements_per_block / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; const __m512 scale_v = _mm512_set1_ps(scale); const __m256 scale_v256 = _mm256_set1_ps(scale); if constexpr (iterations_32 > 0) { #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_32; ++iter) { + for (u32 iter = 0; iter < iterations_32; ++iter) { const __m512 source1 = _mm512_loadu_ps(input); const __m512 source2 = _mm512_loadu_ps(input + 16); @@ -207,7 +208,7 @@ template const __m512i packed = m512::mm512_pack_epi16_avx512vbmi_9to16(make_m512i_from_2x256(converted1, converted2)); mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } } @@ -220,7 +221,7 @@ template const __m256i packed = m256::mm256_pack_epi16_avx512vbmi_9to16(converted); mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -232,7 +233,7 @@ template const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -248,28 +249,27 @@ template return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_17to24(const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_compress_block_avx512vbmi_17to24(const f32* __restrict__ input, const f32 scale, u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_16 = elements_per_block / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; const __m512 scale_v = _mm512_set1_ps(scale); const __m256 scale_v256 = _mm256_set1_ps(scale); if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (uint32_t i = 0; i < iterations_16; ++i) { + for (u32 i = 0; i < iterations_16; ++i) { const __m512 source = _mm512_loadu_ps(input); const __m512i packed_input = mm512_clamp_signed_epi32(mm512_quantize_ps_epi32(source, scale_v)); const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(packed_input); mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } } @@ -281,14 +281,14 @@ template const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m256 source = mm256_loadu_elements_ps(remaining_elements, input); const __m256i packed_input = mm256_clamp_signed_epi32_avx512(mm256_quantize_ps_epi32(source, scale_v256)); - const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); + const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(packed_input); mm256_storeu_elements_epi8(output, tail_bytes(BIT_WIDTH, remaining_elements), packed); } @@ -296,22 +296,21 @@ template return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_1to8(const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_compress_block_avx512vbmi_1to8(const f64* __restrict__ input, const f64 scale, u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_64 = elements_per_block / 64; - constexpr uint32_t iterations_32 = (elements_per_block % 64) / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + constexpr u32 iterations_64 = elements_per_block / 64; + constexpr u32 iterations_32 = (elements_per_block % 64) / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; const __m512d scale_v = _mm512_set1_pd(scale); if constexpr (iterations_64 > 0) { #pragma GCC unroll 8 - for (uint32_t iter = 0; iter < iterations_64; ++iter) { + for (u32 iter = 0; iter < iterations_64; ++iter) { const __m512d source1 = _mm512_loadu_pd(input); const __m512d source2 = _mm512_loadu_pd(input + 8); const __m512d source3 = _mm512_loadu_pd(input + 16); @@ -344,7 +343,7 @@ template mm512_storeu_elements_epi64(output, BIT_WIDTH, packed); - input += 64; + input += 64; output += 8 * BIT_WIDTH; } } @@ -370,7 +369,7 @@ template mm256_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } @@ -387,13 +386,13 @@ template mm_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { - constexpr uint32_t source1_elements = remaining_elements > 8 ? 8 : remaining_elements; - constexpr uint32_t source2_elements = remaining_elements > 8 ? remaining_elements - 8 : 0; + constexpr u32 source1_elements = remaining_elements > 8 ? 8 : remaining_elements; + constexpr u32 source2_elements = remaining_elements > 8 ? remaining_elements - 8 : 0; const __m512d source1 = mm512_loadu_elements_pd(source1_elements, input); const __m512d source2 = source2_elements > 0 ? mm512_loadu_elements_pd(source2_elements, input + 8) : _mm512_setzero_pd(); @@ -411,22 +410,21 @@ template return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_9to16(const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_compress_block_avx512vbmi_9to16(const f64* __restrict__ input, const f64 scale, u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_32 = elements_per_block / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; const __m512d scale_v = _mm512_set1_pd(scale); if constexpr (iterations_32 > 0) { #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_32; ++iter) { + for (u32 iter = 0; iter < iterations_32; ++iter) { const __m512d source1 = _mm512_loadu_pd(input); const __m512d source2 = _mm512_loadu_pd(input + 8); const __m512d source3 = _mm512_loadu_pd(input + 16); @@ -447,7 +445,7 @@ template mm512_storeu_elements_epi32(output, BIT_WIDTH, packed); - input += 32; + input += 32; output += 4 * BIT_WIDTH; } } @@ -465,7 +463,7 @@ template mm256_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } @@ -477,7 +475,7 @@ template const __m128i packed = m128::mm_pack_epi16_avx512vbmi_9to16(converted); mm_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -493,21 +491,20 @@ template return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_compress_block_avx512vbmi_17to24(const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_compress_block_avx512vbmi_17to24(const f64* __restrict__ input, const f64 scale, u8* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_16 = elements_per_block / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; const __m512d scale_v = _mm512_set1_pd(scale); if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (uint32_t i = 0; i < iterations_16; ++i) { + for (u32 i = 0; i < iterations_16; ++i) { const __m512d source1 = _mm512_loadu_pd(input); const __m512d source2 = _mm512_loadu_pd(input + 8); @@ -517,7 +514,7 @@ template const __m512i packed = m512::mm512_pack_epi32_avx512vbmi_17to24(make_m512i_from_2x256(quantized1, quantized2)); mm512_storeu_elements_epi16(output, BIT_WIDTH, packed); - input += 16; + input += 16; output += 2 * BIT_WIDTH; } } @@ -529,7 +526,7 @@ template const __m256i packed = m256::mm256_pack_epi32_avx512vbmi_17to24(quantized); mm256_storeu_elements_epi8(output, BIT_WIDTH, packed); - input += 8; + input += 8; output += BIT_WIDTH; } @@ -543,7 +540,7 @@ template return 0; } -} // namespace internal +} // namespace internal /** * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. @@ -557,9 +554,12 @@ template * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_block_avx512vbmi(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { +int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + std::memset(output, 0, BLOCK_SIZE); if constexpr (BIT_WIDTH <= 8) { @@ -582,9 +582,12 @@ int mm512_compress_block_avx512vbmi(const float_t* __restrict__ input, const flo * * @note This overload is declared for parity with the float path. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_block_avx512vbmi(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { +int mm512_compress_block_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + std::memset(output, 0, BLOCK_SIZE); if constexpr (BIT_WIDTH <= 8) { @@ -609,16 +612,18 @@ int mm512_compress_block_avx512vbmi(const double_t* __restrict__ input, const do * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_blocks_avx512vbmi(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - const float_t* block_input = input; - uint8_t* block_output = output; +int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - for (uint32_t block = 0; block < blocks; ++block) { + const f32* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; ++block) { mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } @@ -635,88 +640,23 @@ int mm512_compress_blocks_avx512vbmi(const float_t* __restrict__ input, const fl * @param blocks number of blocks to compress. * @return int status code. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_compress_blocks_avx512vbmi(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - const double_t* block_input = input; - uint8_t* block_output = output; +int mm512_compress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f64* block_input = input; + u8* block_output = output; - for (uint32_t block = 0; block < blocks; ++block) { + for (u32 block = 0; block < blocks; ++block) { mm512_compress_block_avx512vbmi(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; block_output += BLOCK_SIZE; } return 0; } -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_compress_block_avx512vbmi(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_compress_block_f64_avx512vbmi(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, - uint8_t* __restrict__ output); - -/** - * @brief Compress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_compress_blocks_avx512vbmi(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Compress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_compress_blocks_f64_avx512vbmi(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, - uint8_t* __restrict__ output, uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif +} // namespace pernix #endif // PERNIX_AVX512VBMI_COMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/decompression.h b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h similarity index 59% rename from include/pernix/x86/avx512vbmi/decompression.h rename to include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h index 61280c9..49eb214 100644 --- a/include/pernix/x86/avx512vbmi/decompression.h +++ b/include/pernix/x86/avx512vbmi/avx512vbmi_decompression.h @@ -1,51 +1,98 @@ #ifndef PERNIX_AVX512VBMI_DECOMPRESSION_H #define PERNIX_AVX512VBMI_DECOMPRESSION_H -#include -#include -#include #include +#include +#include +#include +#include #include +using namespace pernix::x86::internal; + namespace pernix { namespace internal { /** * @brief Dequantize sixteen integer values to floats. */ -[[gnu::always_inline]] inline __m512 mm512_dequantize_epi32(const __m512i& input, const __m512& scale) { +__always_inline __m512 mm512_dequantize_epi32(__m512i input, __m512 scale) { const __m512 converted = _mm512_cvtepi32_ps(input); return _mm512_mul_ps(converted, scale); } -[[gnu::always_inline]] inline __m512d mm512_dequantize_epi64(const __m512i& input, const __m512d& scale) { +__always_inline __m512d mm512_dequantize_epi64(__m512i input, __m512d scale) { const __m512d converted = _mm512_cvtepi64_pd(input); return _mm512_mul_pd(converted, scale); } -template +template +__always_inline __m512i mm512_extend_epi8_epi32(const __m128i input) { + if constexpr (SIGN_VALUES) { + return _mm512_cvtepi8_epi32(input); + } else { + return _mm512_cvtepu8_epi32(input); + } +} + +template +__always_inline __m512i mm512_extend_epi8_epi64(const __m128i input) { + if constexpr (SIGN_VALUES) { + return _mm512_cvtepi8_epi64(input); + } else { + return _mm512_cvtepu8_epi64(input); + } +} + +template +__always_inline __m512i mm512_extend_epi16_epi32(const __m256i input) { + if constexpr (SIGN_VALUES) { + return _mm512_cvtepi16_epi32(input); + } else { + return _mm512_cvtepu16_epi32(input); + } +} + +template +__always_inline __m512i mm512_extend_epi16_epi64(const __m128i input) { + if constexpr (SIGN_VALUES) { + return _mm512_cvtepi16_epi64(input); + } else { + return _mm512_cvtepu16_epi64(input); + } +} + +template +__always_inline __m256i mm256_extend_epi16_epi32(const __m128i input) { + if constexpr (SIGN_VALUES) { + return _mm256_cvtepi16_epi32(input); + } else { + return _mm256_cvtepu16_epi32(input); + } +} + +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const uint32_t iterations_64 = elements_per_block / 64; - const uint32_t iterations_32 = (elements_per_block % 64) / 32; - const uint32_t iterations_16 = (elements_per_block % 32) / 16; - const uint32_t remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + const u32 iterations_64 = elements_per_block / 64; + const u32 iterations_32 = (elements_per_block % 64) / 32; + const u32 iterations_16 = (elements_per_block % 32) / 16; + const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; const __m512 scale_v = _mm512_set1_ps(scale); if constexpr (iterations_64 > 0) { #pragma GCC unroll 8 - for (uint32_t i = 0; i < iterations_64; ++i) { + for (u32 i = 0; i < iterations_64; ++i) { const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted1 = _mm512_cvtepi8_epi32(_mm512_castsi512_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 1)); - const __m512i converted3 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 2)); - const __m512i converted4 = _mm512_cvtepi8_epi32(_mm512_extracti64x2_epi64(unpacked, 3)); + const __m512i converted1 = mm512_extend_epi8_epi32(_mm512_castsi512_si128(unpacked)); + const __m512i converted2 = mm512_extend_epi8_epi32(_mm512_extracti64x2_epi64(unpacked, 1)); + const __m512i converted3 = mm512_extend_epi8_epi32(_mm512_extracti64x2_epi64(unpacked, 2)); + const __m512i converted4 = mm512_extend_epi8_epi32(_mm512_extracti64x2_epi64(unpacked, 3)); const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); @@ -58,7 +105,7 @@ template _mm512_storeu_ps(output + 48, dequantized4); output += 64; - input += 8 * BIT_WIDTH; + input += 8 * BIT_WIDTH; } } @@ -66,8 +113,8 @@ template const __m256i source = mm256_loadu_elements_epi32(BIT_WIDTH, input); const __m256i unpacked = m256::mm256_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted1 = _mm512_cvtepi8_epi32(_mm256_castsi256_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi8_epi32(_mm256_extracti128_si256(unpacked, 1)); + const __m512i converted1 = mm512_extend_epi8_epi32(_mm256_castsi256_si128(unpacked)); + const __m512i converted2 = mm512_extend_epi8_epi32(_mm256_extracti128_si256(unpacked, 1)); const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); @@ -76,28 +123,28 @@ template _mm512_storeu_ps(output + 16, dequantized2); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } if constexpr (iterations_16 > 0) { const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted = _mm512_cvtepi8_epi32(unpacked); + const __m512i converted = mm512_extend_epi8_epi32(unpacked); const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted = _mm512_cvtepi8_epi32(unpacked); + const __m512i converted = mm512_extend_epi8_epi32(unpacked); const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); @@ -107,22 +154,21 @@ template return 0; } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_1to8(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_decompress_block_avx512vbmi_1to8(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - const uint32_t iterations_64 = elements_per_block / 64; - const uint32_t iterations_32 = (elements_per_block % 64) / 32; - const uint32_t iterations_16 = (elements_per_block % 32) / 16; - const uint32_t remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; + const u32 iterations_64 = elements_per_block / 64; + const u32 iterations_32 = (elements_per_block % 64) / 32; + const u32 iterations_16 = (elements_per_block % 32) / 16; + const u32 remaining_elements = elements_per_block - iterations_64 * 64 - iterations_32 * 32 - iterations_16 * 16; const __m512d scale_v = _mm512_set1_pd(scale); if constexpr (iterations_64 > 0) { #pragma GCC unroll 8 - for (uint32_t i = 0; i < iterations_64; ++i) { + for (u32 i = 0; i < iterations_64; ++i) { const __m512i source = mm512_loadu_elements_epi64(BIT_WIDTH, input); const __m512i unpacked = m512::mm512_unpack_epi8_avx512vbmi_1to8(source); @@ -131,14 +177,14 @@ template const __m128i extracted3 = _mm512_extracti64x2_epi64(unpacked, 2); const __m128i extracted4 = _mm512_extracti64x2_epi64(unpacked, 3); - const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); - const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); - const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); - const __m512i converted5 = _mm512_cvtepi8_epi64(extracted3); - const __m512i converted6 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted3, 8)); - const __m512i converted7 = _mm512_cvtepi8_epi64(extracted4); - const __m512i converted8 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted4, 8)); + const __m512i converted1 = mm512_extend_epi8_epi64(extracted1); + const __m512i converted2 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted1, 8)); + const __m512i converted3 = mm512_extend_epi8_epi64(extracted2); + const __m512i converted4 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted2, 8)); + const __m512i converted5 = mm512_extend_epi8_epi64(extracted3); + const __m512i converted6 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted3, 8)); + const __m512i converted7 = mm512_extend_epi8_epi64(extracted4); + const __m512i converted8 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted4, 8)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -159,7 +205,7 @@ template _mm512_storeu_pd(output + 56, dequantized8); output += 64; - input += 8 * BIT_WIDTH; + input += 8 * BIT_WIDTH; } if constexpr (iterations_32 > 0) { @@ -169,10 +215,10 @@ template const __m128i extracted1 = _mm256_castsi256_si128(unpacked); const __m128i extracted2 = _mm256_extracti64x2_epi64(unpacked, 1); - const __m512i converted1 = _mm512_cvtepi8_epi64(extracted1); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted1, 8)); - const __m512i converted3 = _mm512_cvtepi8_epi64(extracted2); - const __m512i converted4 = _mm512_cvtepi8_epi64(_mm_srli_si128(extracted2, 8)); + const __m512i converted1 = mm512_extend_epi8_epi64(extracted1); + const __m512i converted2 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted1, 8)); + const __m512i converted3 = mm512_extend_epi8_epi64(extracted2); + const __m512i converted4 = mm512_extend_epi8_epi64(_mm_srli_si128(extracted2, 8)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -185,15 +231,15 @@ template _mm512_storeu_pd(output + 24, dequantized4); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } if constexpr (iterations_16 > 0) { const __m128i source = mm_loadu_elements_epi16(BIT_WIDTH, input); const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); + const __m512i converted1 = mm512_extend_epi8_epi64(unpacked); + const __m512i converted2 = mm512_extend_epi8_epi64(_mm_srli_si128(unpacked, 8)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -202,15 +248,15 @@ template _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); const __m128i unpacked = m128::mm_unpack_epi8_avx512vbmi_1to8(source); - const __m512i converted1 = _mm512_cvtepi8_epi64(unpacked); - const __m512i converted2 = _mm512_cvtepi8_epi64(_mm_srli_si128(unpacked, 8)); + const __m512i converted1 = mm512_extend_epi8_epi64(unpacked); + const __m512i converted2 = mm512_extend_epi8_epi64(_mm_srli_si128(unpacked, 8)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -225,28 +271,27 @@ template return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_32 = elements_per_block / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; const __m512 scale_v = _mm512_set1_ps(scale); const __m256 scale_v256 = _mm256_set1_ps(scale); if constexpr (iterations_32 > 0) { #pragma GCC unroll 4 - for (uint32_t i = 0; i < iterations_32; ++i) { + for (u32 i = 0; i < iterations_32; ++i) { const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted1 = _mm512_cvtepi16_epi32(_mm512_castsi512_si256(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi32(_mm512_extracti32x8_epi32(unpacked, 1)); + const __m512i converted1 = mm512_extend_epi16_epi32(_mm512_castsi512_si256(unpacked)); + const __m512i converted2 = mm512_extend_epi16_epi32(_mm512_extracti32x8_epi32(unpacked, 1)); const __m512 dequantized1 = mm512_dequantize_epi32(converted1, scale_v); const __m512 dequantized2 = mm512_dequantize_epi32(converted2, scale_v); @@ -255,7 +300,7 @@ template _mm512_storeu_ps(output + 16, dequantized2); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } } @@ -263,33 +308,33 @@ template const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted = _mm512_cvtepi16_epi32(unpacked); + const __m512i converted = mm512_extend_epi16_epi32(unpacked); const __m512 dequantized = mm512_dequantize_epi32(converted, scale_v); _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (iterations_8 > 0) { const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); - const __m256i converted = _mm256_cvtepi16_epi32(unpacked); + const __m256i converted = mm256_extend_epi16_epi32(unpacked); const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); _mm256_storeu_ps(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); - const __m256i converted = _mm256_cvtepi16_epi32(unpacked); + const __m256i converted = mm256_extend_epi16_epi32(unpacked); const __m256 dequantized = mm256_dequantize_epi32(converted, scale_v256); mm256_storeu_elements_ps(output, remaining_elements, dequantized); @@ -298,29 +343,28 @@ template return 0; } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_9to16(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_decompress_block_avx512vbmi_9to16(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_32 = elements_per_block / 32; - constexpr uint32_t iterations_16 = (elements_per_block % 32) / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_32 = elements_per_block / 32; + constexpr u32 iterations_16 = (elements_per_block % 32) / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_32 * 32 - iterations_16 * 16 - iterations_8 * 8; const __m512d scale_v = _mm512_set1_pd(scale); if constexpr (iterations_32 > 0) { #pragma GCC unroll 4 - for (uint32_t i = 0; i < iterations_32; ++i) { + for (u32 i = 0; i < iterations_32; ++i) { const __m512i source = mm512_loadu_elements_epi32(BIT_WIDTH, input); const __m512i unpacked = m512::mm512_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted1 = _mm512_cvtepi16_epi64(_mm512_castsi512_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 1)); - const __m512i converted3 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 2)); - const __m512i converted4 = _mm512_cvtepi16_epi64(_mm512_extracti64x2_epi64(unpacked, 3)); + const __m512i converted1 = mm512_extend_epi16_epi64(_mm512_castsi512_si128(unpacked)); + const __m512i converted2 = mm512_extend_epi16_epi64(_mm512_extracti64x2_epi64(unpacked, 1)); + const __m512i converted3 = mm512_extend_epi16_epi64(_mm512_extracti64x2_epi64(unpacked, 2)); + const __m512i converted4 = mm512_extend_epi16_epi64(_mm512_extracti64x2_epi64(unpacked, 3)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -333,7 +377,7 @@ template _mm512_storeu_pd(output + 24, dequantized4); output += 32; - input += 4 * BIT_WIDTH; + input += 4 * BIT_WIDTH; } } @@ -341,8 +385,8 @@ template const __m256i source = mm256_loadu_elements_epi16(BIT_WIDTH, input); const __m256i unpacked = m256::mm256_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted1 = _mm512_cvtepi16_epi64(_mm256_castsi256_si128(unpacked)); - const __m512i converted2 = _mm512_cvtepi16_epi64(_mm256_extracti64x2_epi64(unpacked, 1)); + const __m512i converted1 = mm512_extend_epi16_epi64(_mm256_castsi256_si128(unpacked)); + const __m512i converted2 = mm512_extend_epi16_epi64(_mm256_extracti64x2_epi64(unpacked, 1)); const __m512d dequantized1 = mm512_dequantize_epi64(converted1, scale_v); const __m512d dequantized2 = mm512_dequantize_epi64(converted2, scale_v); @@ -351,28 +395,28 @@ template _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } if constexpr (iterations_8 > 0) { const __m128i source = mm_loadu_elements_epi8(BIT_WIDTH, input); const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted = _mm512_cvtepi16_epi64(unpacked); + const __m512i converted = mm512_extend_epi16_epi64(unpacked); const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); _mm512_storeu_pd(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { const __m128i source = mm_loadu_elements_epi8(tail_bytes(BIT_WIDTH, remaining_elements), input); const __m128i unpacked = m128::mm_unpack_epi16_avx512vbmi_9to16(source); - const __m512i converted = _mm512_cvtepi16_epi64(unpacked); + const __m512i converted = mm512_extend_epi16_epi64(unpacked); const __m512d dequantized = mm512_dequantize_epi64(converted, scale_v); @@ -382,22 +426,21 @@ template return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restrict__ input, const f32 scale, f32* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_16 = elements_per_block / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; const __m512 scale_v = _mm512_set1_ps(scale); const __m256 scale_v256 = _mm256_set1_ps(scale); if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (uint32_t i = 0; i < iterations_16; ++i) { + for (u32 i = 0; i < iterations_16; ++i) { const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24(source); @@ -406,7 +449,7 @@ template _mm512_storeu_ps(output, dequantized); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } } @@ -419,7 +462,7 @@ template _mm256_storeu_ps(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -434,21 +477,20 @@ template return 0; } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -[[gnu::always_inline]] inline int mm512_decompress_block_avx512vbmi_17to24(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; +__always_inline int mm512_decompress_block_avx512vbmi_17to24(const u8* __restrict__ input, const f64 scale, f64* __restrict__ output) { + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_16 = elements_per_block / 16; - constexpr uint32_t iterations_8 = (elements_per_block % 16) / 8; - constexpr uint32_t remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; + constexpr u32 iterations_16 = elements_per_block / 16; + constexpr u32 iterations_8 = (elements_per_block % 16) / 8; + constexpr u32 remaining_elements = elements_per_block - iterations_16 * 16 - iterations_8 * 8; const __m512d scale_v = _mm512_set1_pd(scale); if constexpr (iterations_16 > 0) { #pragma GCC unroll 2 - for (uint32_t i = 0; i < iterations_16; ++i) { + for (u32 i = 0; i < iterations_16; ++i) { const __m512i source = mm512_loadu_elements_epi16(BIT_WIDTH, input); const __m512i unpacked = m512::mm512_unpack_epi32_avx512vbmi_17to24(source); @@ -462,7 +504,7 @@ template _mm512_storeu_pd(output + 8, dequantized2); output += 16; - input += 2 * BIT_WIDTH; + input += 2 * BIT_WIDTH; } } @@ -477,7 +519,7 @@ template _mm512_storeu_pd(output, dequantized); output += 8; - input += BIT_WIDTH; + input += BIT_WIDTH; } if constexpr (remaining_elements > 0) { @@ -493,7 +535,7 @@ template return 0; } -} // namespace internal +} // namespace internal /** * @brief Decompress a single 512\-bit block using AVX-512 and AVX-512-VBMI instructions. @@ -508,10 +550,12 @@ template * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return internal::mm512_decompress_block_avx512vbmi_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { @@ -532,10 +576,12 @@ __always_inline int mm512_decompress_block_avx512vbmi(const uint8_t* __restrict_ * @param output pointer to the output buffer where decompressed double values will be stored. * @return int status code. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -__always_inline int mm512_decompress_block_avx512vbmi(const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { +__always_inline int mm512_decompress_block_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { return internal::mm512_decompress_block_avx512vbmi_1to8(input, scale, output); } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { @@ -560,16 +606,19 @@ __always_inline int mm512_decompress_block_avx512vbmi(const uint8_t* __restrict_ * * @note This function requires AVX-512 and AVX-512-VBMI support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; +int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const u8* block_input = input; + f32* block_output = output; - for (uint32_t block = 0; block < blocks; ++block) { + for (u32 block = 0; block < blocks; ++block) { mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -587,86 +636,23 @@ int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const * @param blocks number of blocks to decompress. * @return int status code. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm512_decompress_blocks_avx512vbmi(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; +int mm512_decompress_blocks_avx512vbmi(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, + const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - for (uint32_t block = 0; block < blocks; ++block) { + const u8* block_input = input; + f64* block_output = output; + + for (u32 block = 0; block < blocks; ++block) { mm512_decompress_block_avx512vbmi(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif -/** - * @brief Decompress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_decompress_block_avx512vbmi(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single 512-bit block using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_decompress_block_f64_avx512vbmi(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output); - -/** - * @brief Decompress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_decompress_blocks_avx512vbmi(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Decompress multiple 512-bit blocks using AVX-512 and AVX-512-VBMI instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed double values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX-512 and AVX-512-VBMI support. - */ -int mm512_decompress_blocks_f64_avx512vbmi(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, - double_t* __restrict__ output, uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif +} // namespace pernix #endif // PERNIX_AVX512VBMI_DECOMPRESSION_H diff --git a/include/pernix/x86/avx512vbmi/compat.h b/include/pernix/x86/avx512vbmi/compat.h index bbc7ad4..05d346e 100644 --- a/include/pernix/x86/avx512vbmi/compat.h +++ b/include/pernix/x86/avx512vbmi/compat.h @@ -2,386 +2,387 @@ #define PERNIX_AVX512_COMPAT_H #include + #include #include #include namespace pernix::internal { -static __always_inline __mmask8 element_mask8(const uint32_t e) { +static __always_inline __mmask8 element_mask8(const u32 e) { return static_cast<__mmask8>(e >= 8 ? 0xFFu : ((1u << e) - 1u)); } -static __always_inline __mmask16 element_mask16(const uint32_t e) { +static __always_inline __mmask16 element_mask16(const u32 e) { return static_cast<__mmask16>(e >= 16 ? 0xFFFFu : ((1u << e) - 1u)); } -static __always_inline __mmask32 element_mask32(const uint32_t e) { +static __always_inline __mmask32 element_mask32(const u32 e) { return e >= 32 ? 0xFFFFFFFFu : (1u << e) - 1u; } -static __always_inline __mmask64 element_mask64(const uint32_t e) { +static __always_inline __mmask64 element_mask64(const u32 e) { return e >= 64 ? 0xFFFFFFFFFFFFFFFFull : (1ull << e) - 1ull; } -static __always_inline __m512i mm512_loadu_elements_epi64(const uint32_t e, const void* mem_addr) { +static __always_inline __m512i mm512_loadu_elements_epi64(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int64_t)); + std::memcpy(&a, mem_addr, e * sizeof(i64)); return a; #else return _mm512_maskz_loadu_epi64(element_mask8(e), mem_addr); #endif } -static __always_inline __m256i mm256_loadu_elements_epi64(const uint32_t e, const void* mem_addr) { +static __always_inline __m256i mm256_loadu_elements_epi64(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int64_t)); + std::memcpy(&a, mem_addr, e * sizeof(i64)); return a; #else return _mm256_maskz_loadu_epi64(element_mask8(e), mem_addr); #endif } -static __always_inline __m128i mm_loadu_elements_epi64(const uint32_t e, const void* mem_addr) { +static __always_inline __m128i mm_loadu_elements_epi64(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int64_t)); + std::memcpy(&a, mem_addr, e * sizeof(i64)); return a; #else return _mm_maskz_loadu_epi64(element_mask8(e), mem_addr); #endif } -static __always_inline __m512i mm512_loadu_elements_epi32(const uint32_t e, const void* mem_addr) { +static __always_inline __m512i mm512_loadu_elements_epi32(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int32_t)); + std::memcpy(&a, mem_addr, e * sizeof(i32)); return a; #else return _mm512_maskz_loadu_epi32(element_mask16(e), mem_addr); #endif } -static __always_inline __m256i mm256_loadu_elements_epi32(const uint32_t e, const void* mem_addr) { +static __always_inline __m256i mm256_loadu_elements_epi32(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int32_t)); + std::memcpy(&a, mem_addr, e * sizeof(i32)); return a; #else return _mm256_maskz_loadu_epi32(element_mask8(e), mem_addr); #endif } -static __always_inline __m128i mm_loadu_elements_epi32(const uint32_t e, const void* mem_addr) { +static __always_inline __m128i mm_loadu_elements_epi32(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int32_t)); + std::memcpy(&a, mem_addr, e * sizeof(i32)); return a; #else return _mm_maskz_loadu_epi32(element_mask8(e), mem_addr); #endif } -static __always_inline __m512i mm512_loadu_elements_epi16(const uint32_t e, const void* mem_addr) { +static __always_inline __m512i mm512_loadu_elements_epi16(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int16_t)); + std::memcpy(&a, mem_addr, e * sizeof(i16)); return a; #else return _mm512_maskz_loadu_epi16(element_mask32(e), mem_addr); #endif } -static __always_inline __m256i mm256_loadu_elements_epi16(const uint32_t e, const void* mem_addr) { +static __always_inline __m256i mm256_loadu_elements_epi16(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int16_t)); + std::memcpy(&a, mem_addr, e * sizeof(i16)); return a; #else return _mm256_maskz_loadu_epi16(element_mask16(e), mem_addr); #endif } -static __always_inline __m128i mm_loadu_elements_epi16(const uint32_t e, const void* mem_addr) { +static __always_inline __m128i mm_loadu_elements_epi16(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int16_t)); + std::memcpy(&a, mem_addr, e * sizeof(i16)); return a; #else return _mm_maskz_loadu_epi16(element_mask8(e), mem_addr); #endif } -static __always_inline __m512i mm512_loadu_elements_epi8(const uint32_t e, const void* mem_addr) { +static __always_inline __m512i mm512_loadu_elements_epi8(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512i a = _mm512_setzero_si512(); - std::memcpy(&a, mem_addr, e * sizeof(int8_t)); + std::memcpy(&a, mem_addr, e * sizeof(i8)); return a; #else return _mm512_maskz_loadu_epi8(element_mask64(e), mem_addr); #endif } -static __always_inline __m256i mm256_loadu_elements_epi8(const uint32_t e, const void* mem_addr) { +static __always_inline __m256i mm256_loadu_elements_epi8(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256i a = _mm256_setzero_si256(); - std::memcpy(&a, mem_addr, e * sizeof(int8_t)); + std::memcpy(&a, mem_addr, e * sizeof(i8)); return a; #else return _mm256_maskz_loadu_epi8(element_mask32(e), mem_addr); #endif } -static __always_inline __m128i mm_loadu_elements_epi8(const uint32_t e, const void* mem_addr) { +static __always_inline __m128i mm_loadu_elements_epi8(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128i a = _mm_setzero_si128(); - std::memcpy(&a, mem_addr, e * sizeof(int8_t)); + std::memcpy(&a, mem_addr, e * sizeof(i8)); return a; #else return _mm_maskz_loadu_epi8(element_mask16(e), mem_addr); #endif } -static __always_inline void mm512_storeu_elements_epi64(void* mem_addr, const uint32_t e, const __m512i a) { +static __always_inline void mm512_storeu_elements_epi64(void* mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; + alignas(64) u8 bytes[64]; _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i64)); #else _mm512_mask_storeu_epi64(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm256_storeu_elements_epi64(void* mem_addr, const uint32_t e, const __m256i a) { +static __always_inline void mm256_storeu_elements_epi64(void* mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; + alignas(32) u8 bytes[32]; _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i64)); #else _mm256_mask_storeu_epi64(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm_storeu_elements_epi64(void* mem_addr, const uint32_t e, const __m128i a) { +static __always_inline void mm_storeu_elements_epi64(void* mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; + alignas(16) u8 bytes[16]; _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int64_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i64)); #else _mm_mask_storeu_epi64(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm512_storeu_elements_epi32(void* mem_addr, const uint32_t e, const __m512i a) { +static __always_inline void mm512_storeu_elements_epi32(void* mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; + alignas(64) u8 bytes[64]; _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i32)); #else _mm512_mask_storeu_epi32(mem_addr, element_mask16(e), a); #endif } -static __always_inline void mm256_storeu_elements_epi32(void* mem_addr, const uint32_t e, const __m256i a) { +static __always_inline void mm256_storeu_elements_epi32(void* mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; + alignas(32) u8 bytes[32]; _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i32)); #else _mm256_mask_storeu_epi32(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm_storeu_elements_epi32(void* mem_addr, const uint32_t e, const __m128i a) { +static __always_inline void mm_storeu_elements_epi32(void* mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; + alignas(16) u8 bytes[16]; _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int32_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i32)); #else _mm_mask_storeu_epi32(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm512_storeu_elements_epi16(void* mem_addr, const uint32_t e, const __m512i a) { +static __always_inline void mm512_storeu_elements_epi16(void* mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; + alignas(64) u8 bytes[64]; _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i16)); #else _mm512_mask_storeu_epi16(mem_addr, element_mask32(e), a); #endif } -static __always_inline void mm256_storeu_elements_epi16(void* mem_addr, const uint32_t e, const __m256i a) { +static __always_inline void mm256_storeu_elements_epi16(void* mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; + alignas(32) u8 bytes[32]; _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i16)); #else _mm256_mask_storeu_epi16(mem_addr, element_mask16(e), a); #endif } -static __always_inline void mm_storeu_elements_epi16(void* mem_addr, const uint32_t e, const __m128i a) { +static __always_inline void mm_storeu_elements_epi16(void* mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; + alignas(16) u8 bytes[16]; _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int16_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i16)); #else _mm_mask_storeu_epi16(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm512_storeu_elements_epi8(void* mem_addr, const uint32_t e, const __m512i a) { +static __always_inline void mm512_storeu_elements_epi8(void* mem_addr, const u32 e, const __m512i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) uint8_t bytes[64]; + alignas(64) u8 bytes[64]; _mm512_storeu_si512(bytes, a); - std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i8)); #else _mm512_mask_storeu_epi8(mem_addr, element_mask64(e), a); #endif } -static __always_inline void mm256_storeu_elements_epi8(void* mem_addr, const uint32_t e, const __m256i a) { +static __always_inline void mm256_storeu_elements_epi8(void* mem_addr, const u32 e, const __m256i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) uint8_t bytes[32]; + alignas(32) u8 bytes[32]; _mm256_storeu_si256(reinterpret_cast<__m256i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i8)); #else _mm256_mask_storeu_epi8(mem_addr, element_mask32(e), a); #endif } -static __always_inline void mm_storeu_elements_epi8(void* mem_addr, const uint32_t e, const __m128i a) { +static __always_inline void mm_storeu_elements_epi8(void* mem_addr, const u32 e, const __m128i a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) uint8_t bytes[16]; + alignas(16) u8 bytes[16]; _mm_storeu_si128(reinterpret_cast<__m128i*>(bytes), a); - std::memcpy(mem_addr, bytes, e * sizeof(int8_t)); + std::memcpy(mem_addr, bytes, e * sizeof(i8)); #else _mm_mask_storeu_epi8(mem_addr, element_mask16(e), a); #endif } -static __always_inline __m512 mm512_loadu_elements_ps(const uint32_t e, const void* mem_addr) { +static __always_inline __m512 mm512_loadu_elements_ps(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512 a = _mm512_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(float_t)); + std::memcpy(&a, mem_addr, e * sizeof(f32)); return a; #else return _mm512_maskz_loadu_ps(element_mask16(e), mem_addr); #endif } -static __always_inline __m256 mm256_loadu_elements_ps(const uint32_t e, const void* mem_addr) { +static __always_inline __m256 mm256_loadu_elements_ps(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256 a = _mm256_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(float_t)); + std::memcpy(&a, mem_addr, e * sizeof(f32)); return a; #else return _mm256_maskz_loadu_ps(element_mask8(e), mem_addr); #endif } -static __always_inline __m128 mm_loadu_elements_ps(const uint32_t e, const void* mem_addr) { +static __always_inline __m128 mm_loadu_elements_ps(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128 a = _mm_setzero_ps(); - std::memcpy(&a, mem_addr, e * sizeof(float_t)); + std::memcpy(&a, mem_addr, e * sizeof(f32)); return a; #else return _mm_maskz_loadu_ps(element_mask8(e), mem_addr); #endif } -static __always_inline __m512d mm512_loadu_elements_pd(const uint32_t e, const void* mem_addr) { +static __always_inline __m512d mm512_loadu_elements_pd(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m512d a = _mm512_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(double_t)); + std::memcpy(&a, mem_addr, e * sizeof(f64)); return a; #else return _mm512_maskz_loadu_pd(element_mask8(e), mem_addr); #endif } -static __always_inline __m256d mm256_loadu_elements_pd(const uint32_t e, const void* mem_addr) { +static __always_inline __m256d mm256_loadu_elements_pd(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m256d a = _mm256_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(double_t)); + std::memcpy(&a, mem_addr, e * sizeof(f64)); return a; #else return _mm256_maskz_loadu_pd(element_mask8(e), mem_addr); #endif } -static __always_inline __m128d mm_loadu_elements_pd(const uint32_t e, const void* mem_addr) { +static __always_inline __m128d mm_loadu_elements_pd(const u32 e, const void* mem_addr) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) __m128d a = _mm_setzero_pd(); - std::memcpy(&a, mem_addr, e * sizeof(double_t)); + std::memcpy(&a, mem_addr, e * sizeof(f64)); return a; #else return _mm_maskz_loadu_pd(element_mask8(e), mem_addr); #endif } -static __always_inline void mm512_storeu_elements_ps(void* mem_addr, const uint32_t e, const __m512 a) { +static __always_inline void mm512_storeu_elements_ps(void* mem_addr, const u32 e, const __m512 a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) float_t values[16]; + alignas(64) f32 values[16]; _mm512_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(float_t)); + std::memcpy(mem_addr, values, e * sizeof(f32)); #else _mm512_mask_storeu_ps(mem_addr, element_mask16(e), a); #endif } -static __always_inline void mm256_storeu_elements_ps(void* mem_addr, const uint32_t e, const __m256 a) { +static __always_inline void mm256_storeu_elements_ps(void* mem_addr, const u32 e, const __m256 a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) float_t values[8]; + alignas(32) f32 values[8]; _mm256_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(float_t)); + std::memcpy(mem_addr, values, e * sizeof(f32)); #else _mm256_mask_storeu_ps(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm_storeu_elements_ps(void* mem_addr, const uint32_t e, const __m128 a) { +static __always_inline void mm_storeu_elements_ps(void* mem_addr, const u32 e, const __m128 a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) float_t values[4]; + alignas(16) f32 values[4]; _mm_storeu_ps(values, a); - std::memcpy(mem_addr, values, e * sizeof(float_t)); + std::memcpy(mem_addr, values, e * sizeof(f32)); #else _mm_mask_storeu_ps(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm512_storeu_elements_pd(void* mem_addr, const uint32_t e, const __m512d a) { +static __always_inline void mm512_storeu_elements_pd(void* mem_addr, const u32 e, const __m512d a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(64) double_t values[8]; + alignas(64) f64 values[8]; _mm512_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(double_t)); + std::memcpy(mem_addr, values, e * sizeof(f64)); #else _mm512_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm256_storeu_elements_pd(void* mem_addr, const uint32_t e, const __m256d a) { +static __always_inline void mm256_storeu_elements_pd(void* mem_addr, const u32 e, const __m256d a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(32) double_t values[4]; + alignas(32) f64 values[4]; _mm256_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(double_t)); + std::memcpy(mem_addr, values, e * sizeof(f64)); #else _mm256_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif } -static __always_inline void mm_storeu_elements_pd(void* mem_addr, const uint32_t e, const __m128d a) { +static __always_inline void mm_storeu_elements_pd(void* mem_addr, const u32 e, const __m128d a) { #if defined(PERNIX_USE_SIMDE) && !defined(SIMDE_X86_AVX512F_NATIVE) - alignas(16) double_t values[2]; + alignas(16) f64 values[2]; _mm_storeu_pd(values, a); - std::memcpy(mem_addr, values, e * sizeof(double_t)); + std::memcpy(mem_addr, values, e * sizeof(f64)); #else _mm_mask_storeu_pd(mem_addr, element_mask8(e), a); #endif } -} +} // namespace pernix::internal -#endif //PERNIX_AVX512_COMPAT_H +#endif // PERNIX_AVX512_COMPAT_H diff --git a/include/pernix/x86/avx512vbmi/packing.h b/include/pernix/x86/avx512vbmi/packing.h index c9f9db9..c07946c 100644 --- a/include/pernix/x86/avx512vbmi/packing.h +++ b/include/pernix/x86/avx512vbmi/packing.h @@ -1,327 +1,339 @@ #ifndef PERNIX_AVX512VBMI_PACKING_H #define PERNIX_AVX512VBMI_PACKING_H -#include +#include +#include #include +#include + +#include +#include namespace pernix::internal { +template +__always_inline Vec combine_primary_contributors(const PackContributor& pack_contributor, const Combine& combine, + std::index_sequence) { + static_assert(Tables::contributor_count > 0); + static_assert(sizeof...(Indices) + 1 == Tables::contributor_count); + + Vec result = pack_contributor(Tables::primary_permutes[0], Tables::primary_shifts[0], Tables::primary_masks[0]); + ((result = combine(result, pack_contributor(Tables::primary_permutes[Indices + 1], Tables::primary_shifts[Indices + 1], + Tables::primary_masks[Indices + 1]))), + ...); + return result; +} + namespace m128 { +constexpr __mmask16 kPackAlternateByteMask16 = 0xAAAAULL; + +__always_inline __m128i mm_pack_sllv_epi8(const __m128i input, const __m128i count) { + const __m128i high_mask = _mm_set1_epi16(static_cast(0xff00u)); + const __m128i low = _mm_sllv_epi16(input, _mm_andnot_si128(high_mask, count)); + const __m128i high = _mm_sllv_epi16(_mm_and_si128(high_mask, input), _mm_srli_epi16(count, 8)); + return _mm_mask_blend_epi8(kPackAlternateByteMask16, low, high); +} + +__always_inline __m128i mm_pack_srlv_epi8(const __m128i input, const __m128i count) { + const __m128i low_mask = _mm_set1_epi16(0x00ff); + const __m128i low = _mm_srlv_epi16(_mm_and_si128(low_mask, input), _mm_and_si128(low_mask, count)); + const __m128i high = _mm_srlv_epi16(input, _mm_srli_epi16(count, 8)); + return _mm_mask_blend_epi8(kPackAlternateByteMask16, low, high); +} + +template +__always_inline __m128i mm_pack_table_avx512vbmi(const __m128i values) { + using tables = detail::pack_table; + using contributor_table = std::array; + const auto pack_contributor = [values](const contributor_table& permute_indices, const contributor_table& shift_counts, + const u64 active_mask) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m128i value = + _mm_maskz_permutexvar_epi8(static_cast<__mmask16>(active_mask), load_table<__m128i>(permute_indices), values); + return mm_pack_sllv_epi8(value, load_table<__m128i>(shift_counts)); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m128i value = + _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(active_mask), load_table<__m128i>(permute_indices), values); + return _mm_sllv_epi16(value, load_table<__m128i>(shift_counts)); + } else { + const __m128i value = + _mm_maskz_permutexvar_epi32(static_cast<__mmask8>(active_mask), load_table<__m128i>(permute_indices), values); + return _mm_sllv_epi32(value, load_table<__m128i>(shift_counts)); + } + }; + + const auto combine = [](__m128i left, __m128i right) { + return _mm_or_si128(left, right); + }; + __m128i result = + combine_primary_contributors(pack_contributor, combine, std::make_index_sequence{}); + + if constexpr (tables::spill_mask != 0) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m128i permuted = + _mm_maskz_permutexvar_epi8(static_cast<__mmask16>(tables::spill_mask), load_table<__m128i>(tables::spill_permute), values); + result = _mm_or_si128(result, mm_pack_srlv_epi8(permuted, load_table<__m128i>(tables::spill_shift))); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m128i permuted = + _mm_maskz_permutexvar_epi16(static_cast<__mmask8>(tables::spill_mask), load_table<__m128i>(tables::spill_permute), values); + result = _mm_or_si128(result, _mm_srlv_epi16(permuted, load_table<__m128i>(tables::spill_shift))); + } else { + const __m128i permuted = + _mm_maskz_permutexvar_epi32(static_cast<__mmask8>(tables::spill_mask), load_table<__m128i>(tables::spill_permute), values); + result = _mm_or_si128(result, _mm_srlv_epi32(permuted, load_table<__m128i>(tables::spill_shift))); + } + } + return result; +} + /** * @brief Pack 8 16-bit values for bit widths 9 through 16 using VBMI. */ -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -[[gnu::always_inline]] inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i& input) { +__always_inline __m128i mm_pack_epi16_avx512vbmi_9to16(const __m128i input) { if constexpr (BIT_WIDTH == 16) { return input; - } else { - using tables = pack_tables_avx512_16; - const __m128i maskv = _mm_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i permuted1 = _mm_permutexvar_epi16(tables::get_permute1(), masked); - const __m128i permuted2 = _mm_permutexvar_epi16(tables::get_permute2(), masked); - - const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm_or_si128(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m128i permuted1 = _mm_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m128i permuted2 = _mm_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m128i permuted3 = _mm_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); - - const __m128i shifted1 = _mm_sllv_epi16(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); - } } + + const __m128i maskv = _mm_set1_epi16(static_cast(detail::low_bit_mask())); + const __m128i masked_values = _mm_and_si128(input, maskv); + return mm_pack_table_avx512vbmi(masked_values); } /** * @brief Pack 16 8-bit values for bit widths 1 through 8 using VBMI. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -[[gnu::always_inline]] inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i& input) { +__always_inline __m128i mm_pack_epi8_avx512vbmi_1to8(const __m128i input) { if constexpr (BIT_WIDTH == 8) { return input; - } else { - const __m128i maskv = _mm_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm_set1_epi16(static_cast(_mm_cmpgt_epi8_mask(masked, _mm_setzero_si128()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m128i shifted = _mm_srli_epi16(masked, 6); - const __m128i combined = _mm_or_si128(masked, shifted); - - const __m128i shifted2 = _mm_srli_epi32(combined, 12); - const __m128i combined2 = _mm_or_si128(shifted2, combined); - - return _mm_cvtepi32_epi8(combined2); - } else if constexpr (BIT_WIDTH == 3) { - const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); - - const __m128i pair6 = _mm_or_si128(even, _mm_srli_epi16(odd, 5)); - const __m128i packed12 = _mm_or_si128(pair6, _mm_srli_epi32(pair6, 10)); - - return m128::mm_pack_epi16_avx512vbmi_9to16<12>(_mm_cvtepi32_epi16(packed12)); - } else if constexpr (BIT_WIDTH == 4) { - const __m128i shifted = _mm_srli_epi16(masked, 4); - const __m128i combined = _mm_or_si128(masked, shifted); - - return _mm_cvtepi16_epi8(combined); - } else { - const __m128i even = _mm_and_si128(masked, _mm_set1_epi16(0x00FF)); - const __m128i odd = _mm_and_si128(masked, _mm_set1_epi16(0xFF00)); - - const __m128i shifted = _mm_or_si128(even, _mm_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } } + + const __m128i maskv = _mm_set1_epi8(static_cast(detail::low_bit_mask())); + const __m128i masked_values = _mm_and_si128(input, maskv); + return mm_pack_table_avx512vbmi(masked_values); } /** * @brief Pack 4 32-bit values for bit widths 17 through 24 using VBMI. */ -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -[[gnu::always_inline]] inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i& input) { - using tables = pack_tables_avx512_24; +__always_inline __m128i mm_pack_epi32_avx512vbmi_17to24(const __m128i input) { + const __m128i maskv = _mm_set1_epi32(static_cast(detail::low_bit_mask())); + const __m128i masked_values = _mm_and_si128(input, maskv); + return mm_pack_table_avx512vbmi(masked_values); +} +} // namespace m128 - const __m128i maskv = _mm_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m128i masked = _mm_and_si128(input, maskv); +namespace m256 { +constexpr __mmask32 kPackAlternateByteMask32 = 0xAAAAAAAAULL; - const __m128 permuted1 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute1()); - const __m128 permuted2 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute2()); - const __m128 permuted3 = _mm_permutevar_ps(_mm_castsi128_ps(masked), tables::get_permute3()); +__always_inline __m256i mm256_pack_sllv_epi8(const __m256i input, const __m256i count) { + const __m256i high_mask = _mm256_set1_epi16(static_cast(0xff00u)); + const __m256i low = _mm256_sllv_epi16(input, _mm256_andnot_si256(high_mask, count)); + const __m256i high = _mm256_sllv_epi16(_mm256_and_si256(high_mask, input), _mm256_srli_epi16(count, 8)); + return _mm256_mask_blend_epi8(kPackAlternateByteMask32, low, high); +} - const __m128i shifted1 = _mm_sllv_epi32(_mm_castps_si128(permuted1), tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi32(_mm_castps_si128(permuted2), tables::get_shift2()); - const __m128i shifted3 = _mm_srlv_epi32(_mm_castps_si128(permuted3), tables::get_shift3()); +__always_inline __m256i mm256_pack_srlv_epi8(const __m256i input, const __m256i count) { + const __m256i low_mask = _mm256_set1_epi16(0x00ff); + const __m256i low = _mm256_srlv_epi16(_mm256_and_si256(low_mask, input), _mm256_and_si256(low_mask, count)); + const __m256i high = _mm256_srlv_epi16(input, _mm256_srli_epi16(count, 8)); + return _mm256_mask_blend_epi8(kPackAlternateByteMask32, low, high); +} - return _mm_or_si128(_mm_or_si128(shifted1, shifted2), shifted3); +template +__always_inline __m256i mm256_pack_table_avx512vbmi(const __m256i values) { + using tables = detail::pack_table; + using contributor_table = std::array; + const auto pack_contributor = [values](const contributor_table& permute_indices, const contributor_table& shift_counts, + const u64 active_mask) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m256i value = + _mm256_maskz_permutexvar_epi8(static_cast<__mmask32>(active_mask), load_table<__m256i>(permute_indices), values); + return mm256_pack_sllv_epi8(value, load_table<__m256i>(shift_counts)); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m256i value = + _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(active_mask), load_table<__m256i>(permute_indices), values); + return _mm256_sllv_epi16(value, load_table<__m256i>(shift_counts)); + } else { + const __m256i value = + _mm256_maskz_permutexvar_epi32(static_cast<__mmask8>(active_mask), load_table<__m256i>(permute_indices), values); + return _mm256_sllv_epi32(value, load_table<__m256i>(shift_counts)); + } + }; + + const auto combine = [](__m256i left, __m256i right) { + return _mm256_or_si256(left, right); + }; + __m256i result = + combine_primary_contributors(pack_contributor, combine, std::make_index_sequence{}); + + if constexpr (tables::spill_mask != 0) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m256i permuted = _mm256_maskz_permutexvar_epi8(static_cast<__mmask32>(tables::spill_mask), + load_table<__m256i>(tables::spill_permute), values); + result = _mm256_or_si256(result, mm256_pack_srlv_epi8(permuted, load_table<__m256i>(tables::spill_shift))); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m256i permuted = _mm256_maskz_permutexvar_epi16(static_cast<__mmask16>(tables::spill_mask), + load_table<__m256i>(tables::spill_permute), values); + result = _mm256_or_si256(result, _mm256_srlv_epi16(permuted, load_table<__m256i>(tables::spill_shift))); + } else { + const __m256i permuted = _mm256_maskz_permutexvar_epi32(static_cast<__mmask8>(tables::spill_mask), + load_table<__m256i>(tables::spill_permute), values); + result = _mm256_or_si256(result, _mm256_srlv_epi32(permuted, load_table<__m256i>(tables::spill_shift))); + } + } + return result; } -} // namespace m128 -namespace m256 { /** * @brief Pack 16 16-bit values for bit widths 9 through 16 using VBMI. */ -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -[[gnu::always_inline]] inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i& input) { +__always_inline __m256i mm256_pack_epi16_avx512vbmi_9to16(const __m256i input) { if constexpr (BIT_WIDTH == 16) { return input; - } else { - using tables = pack_tables_avx512_16; - const __m256i maskv = _mm256_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m256i permuted1 = _mm256_permutexvar_epi16(tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_permutexvar_epi16(tables::get_permute2(), masked); - - const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm256_or_si256(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m256i permuted1 = _mm256_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); - - const __m256i shifted1 = _mm256_sllv_epi16(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); - } } + + const __m256i maskv = _mm256_set1_epi16(static_cast(detail::low_bit_mask())); + const __m256i masked_values = _mm256_and_si256(input, maskv); + return mm256_pack_table_avx512vbmi(masked_values); } /** * @brief Pack 32 8-bit values for bit widths 1 through 8 using VBMI. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -[[gnu::always_inline]] inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i& input) { +__always_inline __m256i mm256_pack_epi8_avx512vbmi_1to8(const __m256i input) { if constexpr (BIT_WIDTH == 8) { return input; - } else { - const __m256i maskv = _mm256_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm256_set1_epi32(static_cast(_mm256_cmpgt_epi8_mask(masked, _mm256_setzero_si256()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m256i shifted = _mm256_srli_epi16(masked, 6); - const __m256i combined = _mm256_or_si256(masked, shifted); - - const __m256i shifted2 = _mm256_srli_epi32(combined, 12); - const __m256i combined2 = _mm256_or_si256(shifted2, combined); - - return _mm256_castsi128_si256(_mm256_cvtepi32_epi8(combined2)); - } else if constexpr (BIT_WIDTH == 3) { - const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); - - const __m256i pair6 = _mm256_or_si256(even, _mm256_srli_epi16(odd, 5)); - const __m256i packed12 = _mm256_or_si256(pair6, _mm256_srli_epi32(pair6, 10)); - - return m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm256_castsi128_si256(_mm256_cvtepi32_epi16(packed12))); - } else if constexpr (BIT_WIDTH == 4) { - const __m256i shifted = _mm256_srli_epi16(masked, 4); - const __m256i combined = _mm256_or_si256(masked, shifted); - - return _mm256_castsi128_si256(_mm256_cvtepi16_epi8(combined)); - } else { - const __m256i even = _mm256_and_si256(masked, _mm256_set1_epi16(0x00FF)); - const __m256i odd = _mm256_and_si256(masked, _mm256_set1_epi16(0xFF00)); - - const __m256i shifted = _mm256_or_si256(even, _mm256_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm256_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } } + + const __m256i maskv = _mm256_set1_epi8(static_cast(detail::low_bit_mask())); + const __m256i masked_values = _mm256_and_si256(input, maskv); + return mm256_pack_table_avx512vbmi(masked_values); } /** * @brief Pack 8 32-bit values for bit widths 17 through 24 using VBMI. */ -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -[[gnu::always_inline]] inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i& input) { - using tables = pack_tables_avx512_24; +__always_inline __m256i mm256_pack_epi32_avx512vbmi_17to24(const __m256i input) { + const __m256i maskv = _mm256_set1_epi32(static_cast(detail::low_bit_mask())); + const __m256i masked_values = _mm256_and_si256(input, maskv); + return mm256_pack_table_avx512vbmi(masked_values); +} +} // namespace m256 - const __m256i maskv = _mm256_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m256i masked = _mm256_and_si256(input, maskv); +namespace m512 { +constexpr __mmask64 kPackAlternateByteMask64 = 0xAAAAAAAAAAAAAAAAULL; - const __m256i permuted1 = _mm256_permutexvar_epi32(tables::get_permute1(), masked); - const __m256i permuted2 = _mm256_permutexvar_epi32(tables::get_permute2(), masked); - const __m256i permuted3 = _mm256_permutexvar_epi32(tables::get_permute3(), masked); +__always_inline __m512i mm512_pack_sllv_epi8(const __m512i input, const __m512i count) { + const __m512i high_mask = _mm512_set1_epi16(static_cast(0xff00u)); + const __m512i low = _mm512_sllv_epi16(input, _mm512_andnot_si512(high_mask, count)); + const __m512i high = _mm512_sllv_epi16(_mm512_and_si512(high_mask, input), _mm512_srli_epi16(count, 8)); + return _mm512_mask_blend_epi8(kPackAlternateByteMask64, low, high); +} - const __m256i shifted1 = _mm256_sllv_epi32(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi32(permuted2, tables::get_shift2()); - const __m256i shifted3 = _mm256_srlv_epi32(permuted3, tables::get_shift3()); +__always_inline __m512i mm512_pack_srlv_epi8(const __m512i input, const __m512i count) { + const __m512i low_mask = _mm512_set1_epi16(0x00ff); + const __m512i low = _mm512_srlv_epi16(_mm512_and_si512(low_mask, input), _mm512_and_si512(low_mask, count)); + const __m512i high = _mm512_srlv_epi16(input, _mm512_srli_epi16(count, 8)); + return _mm512_mask_blend_epi8(kPackAlternateByteMask64, low, high); +} - return _mm256_or_si256(_mm256_or_si256(shifted1, shifted2), shifted3); +template +__always_inline __m512i mm512_pack_table_avx512vbmi(const __m512i values) { + using tables = detail::pack_table; + using contributor_table = std::array; + const auto pack_contributor = [values](const contributor_table& permute_indices, const contributor_table& shift_counts, + const u64 active_mask) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m512i value = + _mm512_maskz_permutexvar_epi8(static_cast<__mmask64>(active_mask), load_table<__m512i>(permute_indices), values); + return mm512_pack_sllv_epi8(value, load_table<__m512i>(shift_counts)); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m512i value = + _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(active_mask), load_table<__m512i>(permute_indices), values); + return _mm512_sllv_epi16(value, load_table<__m512i>(shift_counts)); + } else { + const __m512i value = + _mm512_maskz_permutexvar_epi32(static_cast<__mmask16>(active_mask), load_table<__m512i>(permute_indices), values); + return _mm512_sllv_epi32(value, load_table<__m512i>(shift_counts)); + } + }; + + const auto combine = [](__m512i left, __m512i right) { + return _mm512_or_si512(left, right); + }; + __m512i result = + combine_primary_contributors(pack_contributor, combine, std::make_index_sequence{}); + + if constexpr (tables::spill_mask != 0) { + if constexpr (sizeof(LaneType) == sizeof(i8)) { + const __m512i permuted = _mm512_maskz_permutexvar_epi8(static_cast<__mmask64>(tables::spill_mask), + load_table<__m512i>(tables::spill_permute), values); + result = _mm512_or_si512(result, mm512_pack_srlv_epi8(permuted, load_table<__m512i>(tables::spill_shift))); + } else if constexpr (sizeof(LaneType) == sizeof(i16)) { + const __m512i permuted = _mm512_maskz_permutexvar_epi16(static_cast<__mmask32>(tables::spill_mask), + load_table<__m512i>(tables::spill_permute), values); + result = _mm512_or_si512(result, _mm512_srlv_epi16(permuted, load_table<__m512i>(tables::spill_shift))); + } else { + const __m512i permuted = _mm512_maskz_permutexvar_epi32(static_cast<__mmask16>(tables::spill_mask), + load_table<__m512i>(tables::spill_permute), values); + result = _mm512_or_si512(result, _mm512_srlv_epi32(permuted, load_table<__m512i>(tables::spill_shift))); + } + } + return result; } -} // namespace m256 -namespace m512 { /** * @brief Pack 32 16-bit values for bit widths 9 through 16 using VBMI. */ -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -[[gnu::always_inline]] inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i& input) { +__always_inline __m512i mm512_pack_epi16_avx512vbmi_9to16(const __m512i input) { if constexpr (BIT_WIDTH == 16) { return input; - } else { - using tables = pack_tables_avx512_16; - const __m512i maskv = _mm512_set1_epi16(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - if constexpr (BIT_WIDTH == 12 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m512i permuted1 = _mm512_permutexvar_epi16(tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_permutexvar_epi16(tables::get_permute2(), masked); - - const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_srlv_epi16(permuted2, tables::get_shift2()); - - return _mm512_or_si512(shifted1, shifted2); - } else { - const auto [mask1, mask2, mask3] = tables::get_permute_masks(); - - const __m512i permuted1 = _mm512_maskz_permutexvar_epi16(mask1, tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_maskz_permutexvar_epi16(mask2, tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_maskz_permutexvar_epi16(mask3, tables::get_permute3(), masked); - - const __m512i shifted1 = _mm512_sllv_epi16(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); - const __m512i shifted3 = _mm512_srlv_epi16(permuted3, tables::get_shift3()); - - return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); - } } + + const __m512i maskv = _mm512_set1_epi16(static_cast(detail::low_bit_mask())); + const __m512i masked_values = _mm512_and_si512(input, maskv); + return mm512_pack_table_avx512vbmi(masked_values); } /** * @brief Pack 64 8-bit values for bit widths 1 through 8 using VBMI. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -[[gnu::always_inline]] inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i& input) { +__always_inline __m512i mm512_pack_epi8_avx512vbmi_1to8(const __m512i input) { if constexpr (BIT_WIDTH == 8) { return input; - } else { - const __m512i maskv = _mm512_set1_epi8(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - if constexpr (BIT_WIDTH == 1) { - return _mm512_set1_epi64(static_cast(_mm512_cmpgt_epi8_mask(masked, _mm512_setzero_si512()))); - } else if constexpr (BIT_WIDTH == 2) { - const __m512i shifted = _mm512_srli_epi16(masked, 6); - const __m512i combined = _mm512_or_si512(masked, shifted); - - const __m512i shifted2 = _mm512_srli_epi32(combined, 12); - const __m512i combined2 = _mm512_or_si512(shifted2, combined); - - return _mm512_castsi128_si512(_mm512_cvtepi32_epi8(combined2)); - } else if constexpr (BIT_WIDTH == 3) { - const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); - - const __m512i pair6 = _mm512_or_si512(even, _mm512_srli_epi16(odd, 5)); - const __m512i packed12 = _mm512_or_si512(pair6, _mm512_srli_epi32(pair6, 10)); - - return _mm512_castsi256_si512(m256::mm256_pack_epi16_avx512vbmi_9to16<12>(_mm512_cvtepi32_epi16(packed12))); - } else if constexpr (BIT_WIDTH == 4) { - const __m512i shifted = _mm512_srli_epi16(masked, 4); - const __m512i combined = _mm512_or_si512(masked, shifted); - - return _mm512_castsi256_si512(_mm512_cvtepi16_epi8(combined)); - } else { - const __m512i even = _mm512_and_si512(masked, _mm512_set1_epi16(0x00FF)); - const __m512i odd = _mm512_and_si512(masked, _mm512_set1_epi16(0xFF00)); - - const __m512i shifted = _mm512_or_si512(even, _mm512_srli_epi16(odd, 8 - BIT_WIDTH)); - return mm512_pack_epi16_avx512vbmi_9to16<2 * BIT_WIDTH>(shifted); - } } + + const __m512i maskv = _mm512_set1_epi8(static_cast(detail::low_bit_mask())); + const __m512i masked_values = _mm512_and_si512(input, maskv); + return mm512_pack_table_avx512vbmi(masked_values); } /** * @brief Pack 16 32-bit values for bit widths 17 through 24 using VBMI. */ -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -[[gnu::always_inline]] inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i& input) { - using tables = pack_tables_avx512_24; - - const __m512i maskv = _mm512_set1_epi32(static_cast((1u << BIT_WIDTH) - 1u)); - const __m512i masked = _mm512_and_si512(input, maskv); - - const __m512i permuted1 = _mm512_permutexvar_epi32(tables::get_permute1(), masked); - const __m512i permuted2 = _mm512_permutexvar_epi32(tables::get_permute2(), masked); - const __m512i permuted3 = _mm512_permutexvar_epi32(tables::get_permute3(), masked); - - const __m512i shifted1 = _mm512_sllv_epi32(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi32(permuted2, tables::get_shift2()); - const __m512i shifted3 = _mm512_srlv_epi32(permuted3, tables::get_shift3()); - - return _mm512_or_si512(_mm512_or_si512(shifted1, shifted2), shifted3); +__always_inline __m512i mm512_pack_epi32_avx512vbmi_17to24(const __m512i input) { + const __m512i maskv = _mm512_set1_epi32(static_cast(detail::low_bit_mask())); + const __m512i masked_values = _mm512_and_si512(input, maskv); + return mm512_pack_table_avx512vbmi(masked_values); } -} // namespace m512 -} // namespace pernix::internal +} // namespace m512 +} // namespace pernix::internal #endif // PERNIX_AVX512VBMI_PACKING_H diff --git a/include/pernix/x86/avx512vbmi/tables.h b/include/pernix/x86/avx512vbmi/tables.h index 9115625..045a93a 100644 --- a/include/pernix/x86/avx512vbmi/tables.h +++ b/include/pernix/x86/avx512vbmi/tables.h @@ -3,15 +3,12 @@ #include -#include #include -#include -#include +#include namespace pernix::internal { - -template -[[gnu::always_inline]] static inline Vec load_table(const std::array& table) { +template +static __always_inline Vec load_table(const std::array& table) { static_assert(sizeof(table) >= sizeof(Vec), "table is smaller than requested SIMD vector"); if constexpr (std::is_same_v) { return _mm512_load_si512(static_cast(table.data())); @@ -21,802 +18,6 @@ template return _mm_load_si128(reinterpret_cast(table.data())); } } - -template - requires(N >= 9 && N <= 15) -struct pack_tables_avx512_16 { - alignas(64) inline static constexpr std::array permute1 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 0, 2, 4, 6, - -1, 9, 11, 13, - 15, 16, 18, 20, - 22, -1, 25, 27, - - 29, 31, 32, 34, - 36, 38, -1, 41, - 43, 45, 47, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 0, 2, -1, 5, - 7, 8, 10, -1, - 13, 15, 16, 18, - -1, 21, 23, 24, - - 26, -1, 29, 31, - 32, 34, -1, 37, - 39, 40, 42, -1, - 45, 47, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 0, 2, 3, 5, - 6, -1, 9, -1, - 12, -1, 15, 16, - 18, 19, 21, 22, - - -1, 25, -1, 28, - -1, 31, 32, 34, - 35, 37, 38, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 12) { - return std::array{ - 1, 2, 3, 5, - 6, 7, 9, 10, - 11, 13, 14, 15, - 17, 18, 19, 21, - - 22, 23, 25, 26, - 27, 29, 30, 31, - 33, 34, 35, 37, - 38, 39, 41, 42 - }; - } else if constexpr (N == 13) { - return std::array{ - 0, -1, 3, 4, - 5, -1, -1, 9, - 10, -1, -1, 14, - 15, 16, -1, 19, - - 20, 21, -1, -1, - 25, 26, -1, -1, - 30, 31, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 14) { - return std::array{ - 1, 2, 3, 4, - 5, 6, 7, 9, - 10, 11, 12, 13, - 14, 15, 17, 18, - - 19, 20, 21, 22, - 23, 25, 26, 27, - 28, 29, 30, 31, - -1, -1, -1, -1 - }; - } else if constexpr (N == 15) { - return std::array{ - 1, 2, 3, 4, - 5, 6, 7, 8, - 9, 10, 11, 12, - 13, 14, 15, 17, - - 18, 19, 20, 21, - 22, 23, 24, 25, - 26, 27, 28, 29, - 30, 31, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - alignas(64) inline static constexpr std::array permute2 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 1, 3, 5, 7, - 8, 10, 12, 14, - -1, 17, 19, 21, - 23, 24, 26, 28, - - 30, -1, 33, 35, - 37, 39, 40, 42, - 44, 46, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 1, 3, 4, 6, - -1, 9, 11, 12, - 14, -1, 17, 19, - 20, 22, -1, 25, - - 27, 28, 30, -1, - 33, 35, 36, 38, - -1, 41, 43, 44, - 46, -1, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 1, -1, 4, -1, - 7, 8, 10, 11, - 13, 14, -1, 17, - -1, 20, -1, 23, - - 24, 26, 27, 29, - 30, -1, 33, -1, - 36, -1, 39, 40, - -1, -1, -1, -1 - }; - } else if constexpr (N == 12) { - return std::array{ - 0, 1, 2, 4, - 5, 6, 8, 9, - 10, 12, 13, 14, - 16, 17, 18, 20, - - 21, 22, 24, 25, - 26, 28, 29, 30, - 32, 33, 34, 36, - 37, 38, 40, 41 - }; - } else if constexpr (N == 13) { - return std::array{ - 1, 2, -1, -1, - 6, 7, 8, -1, - 11, 12, 13, -1, - -1, 17, 18, -1, - - -1, 22, 23, 24, - -1, 27, 28, 29, - -1, -1, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 14) { - return std::array{ - 0, 1, 2, 3, - 4, 5, 6, 8, - 9, 10, 11, 12, - 13, 14, 16, 17, - - 18, 19, 20, 21, - 22, 24, 25, 26, - 27, 28, 29, 30, - -1, -1, -1, -1 - }; - } else if constexpr (N == 15) { - return std::array{ - 0, 1, 2, 3, - 4, 5, 6, 7, - 8, 9, 10, 11, - 12, 13, 14, 16, - - 17, 18, 19, 20, - 21, 22, 23, 24, - 25, 26, 27, 28, - 29, 30, -1, -1 - }; - } - // clang-format on - }(); - - alignas(64) inline static constexpr std::array permute3 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - -1, 1, 3, 5, - 7, 8, 10, 12, - 14, -1, 17, 19, - 21, 23, 24, 26, - - 28, 30, -1, 33, - 35, 37, 39, 40, - 42, 44, 46, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - -1, 1, 3, 4, - 6, -1, 9, 11, - 12, 14, -1, 17, - 19, 20, 22, -1, - - 25, 27, 28, 30, - -1, 33, 35, 36, - 38, -1, 41, 43, - 44, 46, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - -1, 1, 2, 4, - 5, 7, 8, 10, - 11, 13, 14, -1, - 17, 18, 20, 21, - - 23, 24, 26, 27, - 29, 30, -1, 33, - 34, 36, 37, 39, - -1, -1, -1, -1 - }; - } else if constexpr (N == 13) { - return std::array{ - -1, 1, 2, 3, - 4, 6, 7, 8, - 9, 11, 12, 13, - 14, -1, 17, 18, - - 19, 20, 22, 23, - 24, 25, 27, 28, - 29, 30, -1, -1, - -1, -1, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - alignas(64) inline static constexpr std::array shift1 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 0, 2, 4, 6, - 0, 1, 3, 5, - 7, 0, 2, 4, - - 6, 0, 1, 3, - 5, 7, 0, 2, - 4, 6, 0, 1, - 3, 5, 7, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 0, 4, 0, 2, - 6, 0, 4, 0, - 2, 6, 0, 4, - 0, 2, 6, 0, - - 4, 0, 2, 6, - 0, 4, 0, 2, - 6, 0, 4, 0, - 2, 6, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 0, 6, 1, 7, - 2, 0, 3, 0, - 4, 0, 5, 0, - 6, 1, 7, 2, - - 0, 3, 0, 4, - 0, 5, 0, 6, - 1, 7, 2, 0, - -1, -1, -1, -1 - }; - } else if constexpr (N == 12) { - return std::array{ - 12, 8, 4, 12, - 8, 4, 12, 8, - 4, 12, 8, 4, - 12, 8, 4, 12, - - 8, 4, 12, 8, - 4, 12, 8, 4, - 12, 8, 4, 12, - 8, 4, 12, 8 - }; - } else if constexpr (N == 13) { - return std::array{ - 0, 0, 7, 4, - 1, 0, 0, 5, - 2, 0, 0, 6, - 3, 0, 0, 7, - - 4, 1, 0, 0, - 5, 2, 0, 0, - 6, 3, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 14) { - return std::array{ - 14, 12, 10, 8, - 6, 4, 2, 14, - 12, 10, 8, 6, - 4, 2, 14, 12, - - 10, 8, 6, 4, - 2, 14, 12, 10, - 8, 6, 4, 2, - -1, -1, -1, -1 - }; - } else if constexpr (N == 15) { - return std::array{ - 15, 14, 13, 12, - 11, 10, 9, 8, - 7, 6, 5, 4, - 3, 2, 1, 15, - - 14, 13, 12, 11, - 10, 9, 8, 7, - 6, 5, 4, 3, - 2, 1, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - alignas(64) inline static constexpr std::array shift2 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 9, 11, 13, 15, - 8, 10, 12, 14, - 8, 9, 11, 13, - 15, 8, 10, 12, - - 14, 8, 9, 11, - 13, 15, 8, 10, - 12, 14, 8, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 10, 14, 8, 12, - 0, 10, 14, 8, - 12, 0, 10, 14, - 8, 12, 0, 10, - - 14, 8, 12, 0, - 10, 14, 8, 12, - 0, 10, 14, 8, - 12, 0, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 11, 8, 12, 8, - 13, 8, 14, 9, - 15, 10, 8, 11, - - 8, 12, 8, 13, - 8, 14, 9, 15, - 10, 8, 11, 8, - 12, 8, 13, 8, - -1, -1, -1, -1 - }; - } else if constexpr (N == 12) { - return std::array{ - 0, 4, 8, 0, - 4, 8, 0, 4, - 8, 0, 4, 8, - 0, 4, 8, 0, - - 4, 8, 0, 4, - 8, 0, 4, 8, - 0, 4, 8, 0, - 4, 8, 0, 4 - }; - } else if constexpr (N == 13) { - return std::array{ - 13, 10, 0, 0, - 14, 11, 8, 0, - 15, 12, 9, 0, - 0, 13, 10, 0, - - 0, 14, 11, 8, - 0, 15, 12, 9, - 0, 0, -1, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 14) { - return std::array{ - 0, 2, 4, 6, - 8, 10, 12, 0, - 2, 4, 6, 8, - 10, 12, 0, 2, - - 4, 6, 8, 10, - 12, 0, 2, 4, - 6, 8, 10, 12, - -1, -1, -1, -1 - }; - } else if constexpr (N == 15) { - return std::array{ - 0, 1, 2, 3, - 4, 5, 6, 7, - 8, 9, 10, 11, - 12, 13, 14, 0, - - 1, 2, 3, 4, - 5, 6, 7, 8, - 9, 10, 11, 12, - 13, 14, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - alignas(64) inline static constexpr std::array shift3 = [] { - // clang-format off - if constexpr (N == 9) { - return std::array{ - 0, 7, 5, 3, - 1, 8, 6, 4, - 2, 0, 7, 5, - 3, 1, 8, 6, - - 4, 2, 0, 7, - 5, 3, 1, 8, - 6, 4, 2, -1, - -1, -1, -1, -1 - }; - } else if constexpr (N == 10) { - return std::array{ - 0, 6, 2, 8, - 4, 0, 6, 2, - 8, 4, 0, 6, - 2, 8, 4, 0, - - 6, 2, 8, 4, - 0, 6, 2, 8, - 4, 0, 6, 2, - 8, 4, -1, -1 - }; - } else if constexpr (N == 11) { - return std::array{ - 0, 5, 10, 4, - 9, 3, 8, 2, - 7, 1, 6, 0, - 5, 10, 4, 9, - - 3, 8, 2, 7, - 1, 6, 0, 5, - 10, 4, 9, 3, - -1, -1, -1, -1 - }; - } else if constexpr (N == 13) { - return std::array{ - 0, 3, 6, 9, - 12, 2, 5, 8, - 11, 1, 4, 7, - 10, 0, 3, 6, - - 9, 12, 2, 5, - 8, 11, 1, 4, - 7, 10, -1, -1, - -1, -1, -1, -1 - }; - } - return std::array{}; - // clang-format on - }(); - - inline static constexpr std::tuple<__mmask32, __mmask32, __mmask32> get_permute_masks() { - // clang-format off - if constexpr (N == 9) { - return { - 0x07BFDFEF, - 0x03FDFEFF, - 0x07FBFDFE - }; - } else if constexpr (N == 10) { - return { - 0x37BDEF7B, - 0x1EF7BDEF, - 0x3DEF7BDE - }; - } else if constexpr (N == 11) { - return { - 0x07EAFD5F, - 0x0D5FABF5, - 0x0FBFF7FE - }; - } else if constexpr (N == 13) { - return { - 0x0333B99D, - 0x00EE6773, - 0x03FFDFFE - }; - } - return {0, 0, 0}; - // clang-format on - } - - [[gnu::always_inline]] static inline Vec get_permute1() { return load_table(permute1); } - [[gnu::always_inline]] static inline Vec get_permute2() { return load_table(permute2); } - [[gnu::always_inline]] static inline Vec get_permute3() { return load_table(permute3); } - - [[gnu::always_inline]] static inline Vec get_shift1() { return load_table(shift1); } - [[gnu::always_inline]] static inline Vec get_shift2() { return load_table(shift2); } - [[gnu::always_inline]] static inline Vec get_shift3() { return load_table(shift3); } -}; - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct pack_tables_avx512_24 { -private: - struct word_plan { - int32_t left_index1 = -1; - int32_t left_index2 = -1; - int32_t right_index = -1; - uint32_t left_shift1 = 32; - uint32_t left_shift2 = 32; - uint32_t right_shift = 32; - }; - - static constexpr word_plan create_plan(const uint32_t idx) { - word_plan plan{}; - - const uint32_t word_start = idx * 32u; - const uint32_t word_end = word_start + 32u; - - uint32_t left_slot = 0; - for (uint32_t input_lane = 0; input_lane < 16; ++input_lane) { - const uint32_t input_start = input_lane * BIT_WIDTH; - const uint32_t input_end = input_start + BIT_WIDTH; - - const uint32_t overlap_start = std::max(word_start, input_start); - const uint32_t overlap_end = std::min(word_end, input_end); - if (overlap_start >= overlap_end) { - continue; - } - - const auto output_bit = static_cast(overlap_start - word_start); - const auto input_bit = static_cast(overlap_start - input_start); - const int32_t delta = output_bit - input_bit; - - if (delta >= 0) { - if (left_slot == 0) { - plan.left_index1 = static_cast(input_lane); - plan.left_shift1 = static_cast(delta); - ++left_slot; - } else { - plan.left_index2 = static_cast(input_lane); - plan.left_shift2 = static_cast(delta); - } - } else { - plan.right_index = static_cast(input_lane); - plan.right_shift = static_cast(-delta); - } - } - - return plan; - } - - static inline constexpr std::array word_plans = [] { - std::array plans{}; - for (uint32_t i = 0; i < 16; ++i) { - plans[i] = create_plan(i); - } - return plans; - }(); - - template - [[gnu::always_inline]] static constexpr std::array make_table(Getter getter) { - std::array values{}; - for (uint32_t i = 0; i < 16; ++i) { - values[i] = getter(word_plans[i]); - } - return values; - } - - alignas(64) static inline constexpr auto permute1 = make_table([](const word_plan& p) { return p.left_index1; }); - - alignas(64) static inline constexpr auto permute2 = make_table([](const word_plan& p) { return p.left_index2; }); - - alignas(64) static inline constexpr auto permute3 = make_table([](const word_plan& p) { return p.right_index; }); - - alignas(64) static inline constexpr auto shift1 = make_table([](const word_plan& p) { return p.left_shift1; }); - - alignas(64) static inline constexpr auto shift2 = make_table([](const word_plan& p) { return p.left_shift2; }); - - alignas(64) static inline constexpr auto shift3 = make_table([](const word_plan& p) { return p.right_shift; }); - -public: - [[gnu::always_inline]] static inline Vec get_permute1() { return load_table(permute1); } - [[gnu::always_inline]] static inline Vec get_permute2() { return load_table(permute2); } - [[gnu::always_inline]] static inline Vec get_permute3() { return load_table(permute3); } - - [[gnu::always_inline]] static inline Vec get_shift1() { return load_table(shift1); } - [[gnu::always_inline]] static inline Vec get_shift2() { return load_table(shift2); } - [[gnu::always_inline]] static inline Vec get_shift3() { return load_table(shift3); } -}; - -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct unpack_tables_avx512_8 { -private: - alignas(64) inline static constexpr std::array permute1 = [] { - std::array table{}; - std::ranges::fill(table, -1); - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - - table[entry] = static_cast(first_byte); - } - - return table; - }(); - - alignas(64) inline static constexpr std::array permute2 = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t bit_offset = bit_start % 8; - - if (bit_offset + BIT_WIDTH > 8) { - table[entry] = static_cast(first_byte + 1); - } - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift1 = [] { - std::array table{}; - - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8; - - table[entry] = static_cast(bit_offset); - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift2 = [] { - std::array table{}; - - for (size_t entry = 0; entry < 64; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - const size_t spill_bits = (bit_offset + BIT_WIDTH > 8u) ? (bit_offset + BIT_WIDTH - 8u) : 0u; - - table[entry] = spill_bits ? static_cast(8 - bit_offset) : 0; - } - - return table; - }(); - -public: - [[gnu::always_inline]] static inline Vec get_permute1() { return load_table(permute1); } - [[gnu::always_inline]] static inline Vec get_permute2() { return load_table(permute2); } - - [[gnu::always_inline]] static inline Vec get_shift1() { return load_table(shift1); } - [[gnu::always_inline]] static inline Vec get_shift2() { return load_table(shift2); } -}; - -template - requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct unpack_tables_avx512_16 { -private: - alignas(64) inline static constexpr std::array permute1 = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t base = entry * 2; - - table[base] = static_cast(first_byte); - table[base + 1] = static_cast(first_byte + 1); - } - - return table; - }(); - - alignas(64) inline static constexpr std::array permute2 = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t first_byte = bit_start / 8; - const size_t bit_offset = bit_start % 8; - const size_t base = entry * 2; - - if (bit_offset + BIT_WIDTH > 16) { - table[base] = static_cast(first_byte + 2); - } - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift1 = [] { - std::array table{}; - - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - - // Right-shift the 16-bit chunk so the value starts at bit 0. - table[entry] = static_cast(bit_offset); - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift2 = [] { - std::array table{}; - for (size_t entry = 0; entry < 32; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_offset = bit_start % 8u; - const size_t spill_bits = (bit_offset + BIT_WIDTH > 16u) ? (bit_offset + BIT_WIDTH - 16u) : 0u; - - // Move spill bits from byte3 to their final bit positions before merge. - table[entry] = spill_bits ? static_cast(16u - bit_offset) : 0; - } - - return table; - }(); - -public: - [[gnu::always_inline]] static inline Vec get_permute1() { return load_table(permute1); } - [[gnu::always_inline]] static inline Vec get_permute2() { return load_table(permute2); } - - [[gnu::always_inline]] static inline Vec get_shift1() { return load_table(shift1); } - [[gnu::always_inline]] static inline Vec get_shift2() { return load_table(shift2); } -}; - -template - requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24 && - (std::is_same_v || std::is_same_v || std::is_same_v)) -struct unpack_tables_avx512_24 { -private: - alignas(64) inline static constexpr std::array permute = [] { - std::array table{}; - std::ranges::fill(table, -1); - - for (size_t entry = 0; entry < 16; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - const size_t bit_end = bit_start + BIT_WIDTH - 1; - - const size_t first_byte = bit_start / 8; - const size_t last_byte = bit_end / 8; - - const size_t base = entry * 4; - - for (size_t byte = first_byte; byte <= last_byte; ++byte) { - table[base + (byte - first_byte)] = static_cast(byte); - } - } - - return table; - }(); - - alignas(64) inline static constexpr std::array shift = [] { - std::array table{}; - - for (size_t entry = 0; entry < 16; ++entry) { - const size_t bit_start = entry * BIT_WIDTH; - table[entry] = static_cast(32u - BIT_WIDTH - (bit_start % 8u)); - } - - return table; - }(); - -public: - [[gnu::always_inline]] static inline Vec get_permute() { return load_table(permute); } - [[gnu::always_inline]] static inline Vec get_shift() { return load_table(shift); } -}; - } // namespace pernix::internal #endif // PERNIX_AVX512VBMI_TABLES_H diff --git a/include/pernix/x86/avx512vbmi/unpacking.h b/include/pernix/x86/avx512vbmi/unpacking.h index 1cfc29c..08c5d0a 100644 --- a/include/pernix/x86/avx512vbmi/unpacking.h +++ b/include/pernix/x86/avx512vbmi/unpacking.h @@ -1,8 +1,9 @@ #ifndef PERNIX_AVX512VBMI_UNPACKING_H #define PERNIX_AVX512VBMI_UNPACKING_H -#include +#include #include +#include namespace pernix::internal { namespace m128 { @@ -16,19 +17,19 @@ __always_inline static __m128i _mm_srlv_epi8(const __m128i a, const __m128i coun } __always_inline static __m128i _mm_sllv_epi8(const __m128i a, const __m128i count) { - const __m128i mask = _mm_set1_epi16(0xff00); + const __m128i mask = _mm_set1_epi16(static_cast(0xff00u)); const __m128i low_half = _mm_sllv_epi16(a, _mm_andnot_si128(mask, count)); const __m128i high_half = _mm_sllv_epi16(_mm_and_si128(mask, a), _mm_srli_epi16(count, 8)); return _mm_mask_blend_epi8(kAlternateByteMask16, low_half, high_half); } -__always_inline static __m128i _mm_slli_epi8(const __m128i a, const int8_t imm8) { +__always_inline static __m128i _mm_slli_epi8(const __m128i a, const i8 imm8) { return _mm_sllv_epi8(a, _mm_set1_epi8(imm8)); } __always_inline static __m128i _mm_srli_epi8(const __m128i a, const int imm8) { const __m128i lo_mask = _mm_set1_epi16(0x00ff); - const __m128i hi_mask = _mm_set1_epi16(0xff00); + const __m128i hi_mask = _mm_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m128i lo = _mm_srl_epi16(_mm_and_si128(a, lo_mask), shift); @@ -37,9 +38,9 @@ __always_inline static __m128i _mm_srli_epi8(const __m128i a, const int imm8) { return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); } -__always_inline static __m128i _mm_srai_epi8(const __m128i a, const int8_t imm8) { +__always_inline static __m128i _mm_srai_epi8(const __m128i a, const i8 imm8) { const __m128i lo_mask = _mm_set1_epi16(0x00ff); - const __m128i hi_mask = _mm_set1_epi16(0xff00); + const __m128i hi_mask = _mm_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m128i hi = _mm_and_si128(_mm_sra_epi16(a, shift), hi_mask); @@ -50,96 +51,59 @@ __always_inline static __m128i _mm_srai_epi8(const __m128i a, const int8_t imm8) return _mm_mask_blend_epi8(kAlternateByteMask16, lo, hi); } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(const __m128i& input) { +__always_inline __m128i mm_unpack_epi8_avx512vbmi_1to8(__m128i input) { if constexpr (BIT_WIDTH == 8) { return input; } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask16>(_mm_cvtsi128_si64(input)); - const __m128i source = _mm_movm_epi8(value); - const __m128i unpacked = _mm_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m128i values_shift0 = input; - __m128i values_shift2 = _mm_srli_epi16(values_shift0, 2); - const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); - const __m128i values_shift6 = _mm_srli_epi16(values_shift0, 6); - - __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - - interleave_tmp = _mm_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm_unpacklo_epi64(interleave_tmp, values_shift2); - - interleave_tmp = _mm_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); - - values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0303)); - - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m128i values_shift0 = input; - const __m128i values_shift4 = _mm_srli_epi16(values_shift0, 4); - - const __m128i interleave_tmp = _mm_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm_unpacklo_epi64(interleave_tmp, values_shift0); - values_shift0 = _mm_shuffle_epi32(values_shift0, 0xD8); - - values_shift0 = _mm_and_si128(values_shift0, _mm_set1_epi16(0x0F0F)); - - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m128i permuted1 = _mm_permutexvar_epi8(tables::get_permute1(), input); - const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); - - const __m128i shifted1 = _mm_srlv_epi8(permuted1, tables::get_shift1()); - const __m128i shifted2 = _mm_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask16 spill_mask = _mm_cmpneq_epi8_mask(tables::get_shift2(), _mm_setzero_si128()); - __m128i combined = _mm_or_si128(shifted1, _mm_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr uint32_t shift = 8 - BIT_WIDTH; - combined = _mm_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm_srai_epi8(combined, shift); - } else { - combined = _mm_srli_epi8(combined, shift); - } + using tables = detail::unpack_table; + + const __m128i permuted = _mm_permutexvar_epi8(load_table<__m128i>(tables::primary_permute), input); + const __m128i right_shift = load_table<__m128i>(tables::right_shift_magnitude); + __m128i combined = _mm_srlv_epi8(permuted, right_shift); + + if constexpr (tables::has_spill) { + const __m128i spill_permuted = _mm_permutexvar_epi8(load_table<__m128i>(tables::spill_permute), input); + const __m128i spill_shift = load_table<__m128i>(tables::left_shift_for_spill); + const __m128i spill_values = _mm_sllv_epi8(spill_permuted, spill_shift); + const __mmask16 spill_mask = _mm_cmpneq_epi8_mask(spill_shift, _mm_setzero_si128()); + combined = _mm_or_si128(combined, _mm_maskz_mov_epi8(spill_mask, spill_values)); + } - return combined; + constexpr u32 shift = 8 - BIT_WIDTH; + combined = _mm_slli_epi8(combined, shift); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + combined = _mm_srai_epi8(combined, shift); + } else { + combined = _mm_srli_epi8(combined, shift); } + + return combined; } } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i& input) { +__always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(__m128i input) { if constexpr (BIT_WIDTH == 16) { return input; } else { - using tables = unpack_tables_avx512_16; + using tables = detail::unpack_table; - const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute1(), input); + const __m128i permuted = _mm_permutexvar_epi8(load_table<__m128i>(tables::primary_permute), input); - __m128i shifted = _mm_srlv_epi16(permuted, tables::get_shift1()); + const __m128i right_shift = load_table<__m128i>(tables::right_shift_magnitude); + __m128i shifted = _mm_srlv_epi16(permuted, right_shift); - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m128i permuted2 = _mm_permutexvar_epi8(tables::get_permute2(), input); - const __m128i shifted2 = _mm_sllv_epi16(permuted2, tables::get_shift2()); + if constexpr (tables::has_spill) { + const __m128i permuted2 = _mm_permutexvar_epi8(load_table<__m128i>(tables::spill_permute), input); + const __m128i shifted2 = _mm_sllv_epi16(permuted2, load_table<__m128i>(tables::left_shift_for_spill)); shifted = _mm_or_si128(shifted, shifted2); } - constexpr uint32_t shift = 16 - BIT_WIDTH; - shifted = _mm_slli_epi16(shifted, shift); + constexpr u32 shift = 16 - BIT_WIDTH; + shifted = _mm_slli_epi16(shifted, shift); if (SIGN_VALUES) { shifted = _mm_srai_epi16(shifted, shift); } else { @@ -150,15 +114,17 @@ __always_inline __m128i mm_unpack_epi16_avx512vbmi_9to16(const __m128i& input) { } } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i& input) { - using tables = unpack_tables_avx512_24; +__always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(__m128i input) { + using tables = detail::unpack_table; - const __m128i permuted = _mm_permutexvar_epi8(tables::get_permute(), input); + const __m128i permuted = _mm_permutexvar_epi8(load_table<__m128i>(tables::primary_permute), input); + const __m128i right_shift = load_table<__m128i>(tables::right_shift_magnitude); - constexpr uint32_t shift = 32 - BIT_WIDTH; - __m128i shifted = _mm_sllv_epi32(permuted, tables::get_shift()); + constexpr u32 shift = 32 - BIT_WIDTH; + __m128i shifted = _mm_srlv_epi32(permuted, right_shift); + shifted = _mm_slli_epi32(shifted, shift); if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { shifted = _mm_srai_epi32(shifted, shift); } else { @@ -167,7 +133,7 @@ __always_inline __m128i mm_unpack_epi32_avx512vbmi_17to24(const __m128i& input) return shifted; } -} // namespace m128 +} // namespace m128 namespace m256 { constexpr __mmask32 kAlternateByteMask32 = 0xAAAAAAAAULL; @@ -180,19 +146,19 @@ __always_inline static __m256i _mm256_srlv_epi8(const __m256i a, const __m256i c } __always_inline static __m256i _mm256_sllv_epi8(const __m256i a, const __m256i count) { - const __m256i mask = _mm256_set1_epi16(0xff00); + const __m256i mask = _mm256_set1_epi16(static_cast(0xff00u)); const __m256i low_half = _mm256_sllv_epi16(a, _mm256_andnot_si256(mask, count)); const __m256i high_half = _mm256_sllv_epi16(_mm256_and_si256(mask, a), _mm256_srli_epi16(count, 8)); return _mm256_mask_blend_epi8(kAlternateByteMask32, low_half, high_half); } -__always_inline static __m256i _mm256_slli_epi8(const __m256i a, const int8_t imm8) { +__always_inline static __m256i _mm256_slli_epi8(const __m256i a, const i8 imm8) { return _mm256_sllv_epi8(a, _mm256_set1_epi8(imm8)); } -__always_inline static __m256i _mm256_srli_epi8(const __m256i a, const int8_t imm8) { +__always_inline static __m256i _mm256_srli_epi8(const __m256i a, const i8 imm8) { const __m256i lo_mask = _mm256_set1_epi16(0x00ff); - const __m256i hi_mask = _mm256_set1_epi16(0xff00); + const __m256i hi_mask = _mm256_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m256i lo = _mm256_srl_epi16(_mm256_and_si256(a, lo_mask), shift); @@ -201,9 +167,9 @@ __always_inline static __m256i _mm256_srli_epi8(const __m256i a, const int8_t im return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); } -__always_inline static __m256i _mm256_srai_epi8(const __m256i a, const int8_t imm8) { +__always_inline static __m256i _mm256_srai_epi8(const __m256i a, const i8 imm8) { const __m256i lo_mask = _mm256_set1_epi16(0x00ff); - const __m256i hi_mask = _mm256_set1_epi16(0xff00); + const __m256i hi_mask = _mm256_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m256i hi = _mm256_and_si256(_mm256_sra_epi16(a, shift), hi_mask); @@ -214,96 +180,59 @@ __always_inline static __m256i _mm256_srai_epi8(const __m256i a, const int8_t im return _mm256_mask_blend_epi8(kAlternateByteMask32, lo, hi); } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(const __m256i& input) { +__always_inline __m256i mm256_unpack_epi8_avx512vbmi_1to8(__m256i input) { if constexpr (BIT_WIDTH == 8) { return input; } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask32>(_mm_cvtsi128_si64(_mm256_castsi256_si128(input))); - const __m256i source = _mm256_movm_epi8(value); - const __m256i unpacked = _mm256_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m256i values_shift0 = input; - __m256i values_shift2 = _mm256_srli_epi16(values_shift0, 2); - const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); - const __m256i values_shift6 = _mm256_srli_epi16(values_shift0, 6); - - __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); - - interleave_tmp = _mm256_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm256_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm256_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); - - interleave_tmp = _mm256_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm256_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); - values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); - - values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0303)); - - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m256i values_shift0 = input; - const __m256i values_shift4 = _mm256_srli_epi16(values_shift0, 4); - - __m256i interleave_tmp = _mm256_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm256_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm256_shuffle_i32x4(interleave_tmp, values_shift0, 0b00); - values_shift0 = _mm256_shuffle_i32x4(values_shift0, values_shift0, 0b00); - - values_shift0 = _mm256_and_si256(values_shift0, _mm256_set1_epi16(0x0F0F)); - - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m256i permuted1 = _mm256_permutexvar_epi8(tables::get_permute1(), input); - const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); - - const __m256i shifted1 = _mm256_srlv_epi8(permuted1, tables::get_shift1()); - const __m256i shifted2 = _mm256_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask32 spill_mask = _mm256_cmpneq_epi8_mask(tables::get_shift2(), _mm256_setzero_si256()); - __m256i combined = _mm256_or_si256(shifted1, _mm256_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr uint32_t shift = 8 - BIT_WIDTH; - combined = _mm256_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm256_srai_epi8(combined, shift); - } else { - combined = _mm256_srli_epi8(combined, shift); - } + using tables = detail::unpack_table; + + const __m256i permuted = _mm256_permutexvar_epi8(load_table<__m256i>(tables::primary_permute), input); + const __m256i right_shift = load_table<__m256i>(tables::right_shift_magnitude); + __m256i combined = _mm256_srlv_epi8(permuted, right_shift); + + if constexpr (tables::has_spill) { + const __m256i spill_permuted = _mm256_permutexvar_epi8(load_table<__m256i>(tables::spill_permute), input); + const __m256i spill_shift = load_table<__m256i>(tables::left_shift_for_spill); + const __m256i spill_values = _mm256_sllv_epi8(spill_permuted, spill_shift); + const __mmask32 spill_mask = _mm256_cmpneq_epi8_mask(spill_shift, _mm256_setzero_si256()); + combined = _mm256_or_si256(combined, _mm256_maskz_mov_epi8(spill_mask, spill_values)); + } - return combined; + constexpr u32 shift = 8 - BIT_WIDTH; + combined = _mm256_slli_epi8(combined, shift); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + combined = _mm256_srai_epi8(combined, shift); + } else { + combined = _mm256_srli_epi8(combined, shift); } + + return combined; } } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i& input) { +__always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(__m256i input) { if constexpr (BIT_WIDTH == 16) { return input; } else { - using tables = unpack_tables_avx512_16; + using tables = detail::unpack_table; - const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute1(), input); + const __m256i permuted = _mm256_permutexvar_epi8(load_table<__m256i>(tables::primary_permute), input); - __m256i shifted = _mm256_srlv_epi16(permuted, tables::get_shift1()); + const __m256i right_shift = load_table<__m256i>(tables::right_shift_magnitude); + __m256i shifted = _mm256_srlv_epi16(permuted, right_shift); - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m256i permuted2 = _mm256_permutexvar_epi8(tables::get_permute2(), input); - const __m256i shifted2 = _mm256_sllv_epi16(permuted2, tables::get_shift2()); + if constexpr (tables::has_spill) { + const __m256i permuted2 = _mm256_permutexvar_epi8(load_table<__m256i>(tables::spill_permute), input); + const __m256i shifted2 = _mm256_sllv_epi16(permuted2, load_table<__m256i>(tables::left_shift_for_spill)); shifted = _mm256_or_si256(shifted, shifted2); } - constexpr uint32_t shift = 16 - BIT_WIDTH; - shifted = _mm256_slli_epi16(shifted, shift); + constexpr u32 shift = 16 - BIT_WIDTH; + shifted = _mm256_slli_epi16(shifted, shift); if (SIGN_VALUES) { shifted = _mm256_srai_epi16(shifted, shift); } else { @@ -314,15 +243,17 @@ __always_inline __m256i mm256_unpack_epi16_avx512vbmi_9to16(const __m256i& input } } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i& input) { - using tables = unpack_tables_avx512_24; +__always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(__m256i input) { + using tables = detail::unpack_table; - const __m256i permuted = _mm256_permutexvar_epi8(tables::get_permute(), input); + const __m256i permuted = _mm256_permutexvar_epi8(load_table<__m256i>(tables::primary_permute), input); + const __m256i right_shift = load_table<__m256i>(tables::right_shift_magnitude); - constexpr uint32_t shift = 32 - BIT_WIDTH; - __m256i shifted = _mm256_sllv_epi32(permuted, tables::get_shift()); + constexpr u32 shift = 32 - BIT_WIDTH; + __m256i shifted = _mm256_srlv_epi32(permuted, right_shift); + shifted = _mm256_slli_epi32(shifted, shift); if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { shifted = _mm256_srai_epi32(shifted, shift); } else { @@ -331,7 +262,7 @@ __always_inline __m256i mm256_unpack_epi32_avx512vbmi_17to24(const __m256i& inpu return shifted; } -} // namespace m256 +} // namespace m256 namespace m512 { constexpr __mmask64 kAlternateByteMask64 = 0xAAAAAAAAAAAAAAAAULL; @@ -344,19 +275,19 @@ __always_inline static __m512i _mm512_srlv_epi8(const __m512i a, const __m512i c } __always_inline static __m512i _mm512_sllv_epi8(const __m512i a, const __m512i count) { - const __m512i mask = _mm512_set1_epi16(0xff00); + const __m512i mask = _mm512_set1_epi16(static_cast(0xff00u)); const __m512i low_half = _mm512_sllv_epi16(a, _mm512_andnot_si512(mask, count)); const __m512i high_half = _mm512_sllv_epi16(_mm512_and_si512(mask, a), _mm512_srli_epi16(count, 8)); return _mm512_mask_blend_epi8(kAlternateByteMask64, low_half, high_half); } -__always_inline static __m512i _mm512_slli_epi8(const __m512i a, const int8_t imm8) { +__always_inline static __m512i _mm512_slli_epi8(const __m512i a, const i8 imm8) { return _mm512_sllv_epi8(a, _mm512_set1_epi8(imm8)); } -__always_inline static __m512i _mm512_srli_epi8(const __m512i a, const int8_t imm8) { +__always_inline static __m512i _mm512_srli_epi8(const __m512i a, const i8 imm8) { const __m512i lo_mask = _mm512_set1_epi16(0x00ff); - const __m512i hi_mask = _mm512_set1_epi16(0xff00); + const __m512i hi_mask = _mm512_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m512i lo = _mm512_srl_epi16(_mm512_and_si512(a, lo_mask), shift); @@ -365,9 +296,9 @@ __always_inline static __m512i _mm512_srli_epi8(const __m512i a, const int8_t im return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); } -__always_inline static __m512i _mm512_srai_epi8(const __m512i a, const int8_t imm8) { +__always_inline static __m512i _mm512_srai_epi8(const __m512i a, const i8 imm8) { const __m512i lo_mask = _mm512_set1_epi16(0x00ff); - const __m512i hi_mask = _mm512_set1_epi16(0xff00); + const __m512i hi_mask = _mm512_set1_epi16(static_cast(0xff00u)); const __m128i shift = _mm_cvtsi32_si128(imm8); const __m512i hi = _mm512_and_si512(_mm512_sra_epi16(a, shift), hi_mask); @@ -378,95 +309,58 @@ __always_inline static __m512i _mm512_srai_epi8(const __m512i a, const int8_t im return _mm512_mask_blend_epi8(kAlternateByteMask64, lo, hi); } -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 8) -__always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(const __m512i& input) { +__always_inline __m512i mm512_unpack_epi8_avx512vbmi_1to8(__m512i input) { if constexpr (BIT_WIDTH == 8) { return input; } else { - if constexpr (BIT_WIDTH == 1) { - const auto value = static_cast<__mmask64>(_mm_cvtsi128_si64(_mm512_castsi512_si128(input))); - const __m512i source = _mm512_movm_epi8(value); - const __m512i unpacked = _mm512_abs_epi8(source); - return unpacked; - } else if constexpr (BIT_WIDTH == 2) { - __m512i values_shift0 = input; - __m512i values_shift2 = _mm512_srli_epi16(values_shift0, 2); - const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); - const __m512i values_shift6 = _mm512_srli_epi16(values_shift0, 6); - - __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift2); - values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift2); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0b00000000); - - interleave_tmp = _mm512_unpacklo_epi8(values_shift4, values_shift6); - values_shift2 = _mm512_unpackhi_epi8(values_shift4, values_shift6); - values_shift2 = _mm512_shuffle_i32x4(interleave_tmp, values_shift2, 0b00000000); - - interleave_tmp = _mm512_unpacklo_epi16(values_shift0, values_shift2); - values_shift0 = _mm512_unpackhi_epi16(values_shift0, values_shift2); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x88); - values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); - - values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0303)); - - return values_shift0; - } else if constexpr (BIT_WIDTH == 4) { - __m512i values_shift0 = input; - const __m512i values_shift4 = _mm512_srli_epi16(values_shift0, 4); - - __m512i interleave_tmp = _mm512_unpacklo_epi8(values_shift0, values_shift4); - values_shift0 = _mm512_unpackhi_epi8(values_shift0, values_shift4); - values_shift0 = _mm512_shuffle_i32x4(interleave_tmp, values_shift0, 0x44); - values_shift0 = _mm512_shuffle_i32x4(values_shift0, values_shift0, 0xD8); - - values_shift0 = _mm512_and_si512(values_shift0, _mm512_set1_epi16(0x0F0F)); - - return values_shift0; - } else { - using tables = unpack_tables_avx512_8; - - const __m512i permuted1 = _mm512_permutexvar_epi8(tables::get_permute1(), input); - const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); - - const __m512i shifted1 = _mm512_srlv_epi8(permuted1, tables::get_shift1()); - const __m512i shifted2 = _mm512_sllv_epi8(permuted2, tables::get_shift2()); - - const __mmask64 spill_mask = _mm512_cmpneq_epi8_mask(tables::get_shift2(), _mm512_setzero_si512()); - __m512i combined = _mm512_or_si512(shifted1, _mm512_maskz_mov_epi8(spill_mask, shifted2)); - - constexpr uint32_t shift = 8 - BIT_WIDTH; - combined = _mm512_slli_epi8(combined, shift); - if (SIGN_VALUES) { - combined = _mm512_srai_epi8(combined, shift); - } else { - combined = _mm512_srli_epi8(combined, shift); - } + using tables = detail::unpack_table; + + const __m512i permuted = _mm512_permutexvar_epi8(load_table<__m512i>(tables::primary_permute), input); + const __m512i right_shift = load_table<__m512i>(tables::right_shift_magnitude); + __m512i combined = _mm512_srlv_epi8(permuted, right_shift); + + if constexpr (tables::has_spill) { + const __m512i spill_permuted = _mm512_permutexvar_epi8(load_table<__m512i>(tables::spill_permute), input); + const __m512i spill_shift = load_table<__m512i>(tables::left_shift_for_spill); + const __m512i spill_values = _mm512_sllv_epi8(spill_permuted, spill_shift); + const __mmask64 spill_mask = _mm512_cmpneq_epi8_mask(spill_shift, _mm512_setzero_si512()); + combined = _mm512_or_si512(combined, _mm512_maskz_mov_epi8(spill_mask, spill_values)); + } - return combined; + constexpr u32 shift = 8 - BIT_WIDTH; + combined = _mm512_slli_epi8(combined, shift); + if constexpr (SIGN_VALUES && BIT_WIDTH > 1) { + combined = _mm512_srai_epi8(combined, shift); + } else { + combined = _mm512_srli_epi8(combined, shift); } + + return combined; } } -template +template requires(BIT_WIDTH >= 9 && BIT_WIDTH <= 16) -__always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i& input) { +__always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(__m512i input) { if constexpr (BIT_WIDTH == 16) { return input; } else { - using tables = unpack_tables_avx512_16; + using tables = detail::unpack_table; - const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute1(), input); - __m512i shifted = _mm512_srlv_epi16(permuted, tables::get_shift1()); + const __m512i permuted = _mm512_permutexvar_epi8(load_table<__m512i>(tables::primary_permute), input); + const __m512i right_shift = load_table<__m512i>(tables::right_shift_magnitude); + __m512i shifted = _mm512_srlv_epi16(permuted, right_shift); - if constexpr (BIT_WIDTH == 11 || BIT_WIDTH == 13 || BIT_WIDTH == 14 || BIT_WIDTH == 15) { - const __m512i permuted2 = _mm512_permutexvar_epi8(tables::get_permute2(), input); - const __m512i shifted2 = _mm512_sllv_epi16(permuted2, tables::get_shift2()); + if constexpr (tables::has_spill) { + const __m512i permuted2 = _mm512_permutexvar_epi8(load_table<__m512i>(tables::spill_permute), input); + const __m512i shifted2 = _mm512_sllv_epi16(permuted2, load_table<__m512i>(tables::left_shift_for_spill)); shifted = _mm512_or_si512(shifted, shifted2); } - constexpr uint32_t shift = 16 - BIT_WIDTH; - shifted = _mm512_slli_epi16(shifted, shift); + constexpr u32 shift = 16 - BIT_WIDTH; + shifted = _mm512_slli_epi16(shifted, shift); if (SIGN_VALUES) { shifted = _mm512_srai_epi16(shifted, shift); } else { @@ -477,15 +371,17 @@ __always_inline __m512i mm512_unpack_epi16_avx512vbmi_9to16(const __m512i& input } } -template +template requires(BIT_WIDTH >= 17 && BIT_WIDTH <= 24) -__always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i& input) { - using tables = unpack_tables_avx512_24; +__always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(__m512i input) { + using tables = detail::unpack_table; - const __m512i permuted = _mm512_permutexvar_epi8(tables::get_permute(), input); - __m512i shifted = _mm512_sllv_epi32(permuted, tables::get_shift()); + const __m512i permuted = _mm512_permutexvar_epi8(load_table<__m512i>(tables::primary_permute), input); + const __m512i right_shift = load_table<__m512i>(tables::right_shift_magnitude); + __m512i shifted = _mm512_srlv_epi32(permuted, right_shift); - constexpr uint32_t shift = 32 - BIT_WIDTH; + constexpr u32 shift = 32 - BIT_WIDTH; + shifted = _mm512_slli_epi32(shifted, shift); if constexpr (SIGN_VALUES) { shifted = _mm512_srai_epi32(shifted, shift); } else { @@ -494,7 +390,7 @@ __always_inline __m512i mm512_unpack_epi32_avx512vbmi_17to24(const __m512i& inpu return shifted; } -} // namespace m512 -} // namespace pernix::internal +} // namespace m512 +} // namespace pernix::internal #endif // PERNIX_AVX512VBMI_UNPACKING_H diff --git a/include/pernix/x86/bmi2/bmi2_compression.h b/include/pernix/x86/bmi2/bmi2_compression.h new file mode 100644 index 0000000..f4f4584 --- /dev/null +++ b/include/pernix/x86/bmi2/bmi2_compression.h @@ -0,0 +1,323 @@ +#ifndef PERNIX_BMI2_COMPRESSION_H +#define PERNIX_BMI2_COMPRESSION_H + +#include +#include +#include + +#include +#include +#include + +namespace pernix { +namespace internal { +/** + * @brief Build the masks and shift constants used by the BMI2 packers. + * + * @tparam BIT_WIDTH bit width per packed value. + * @return std::tuple mask tuple used by the BMI2 helpers. + */ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) +static constexpr std::tuple pack_avx2_bmi2_constants() { + u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + u64 pext_mask; + u16 shift1 = BIT_WIDTH * 4; + u16 shift2 = 64 - shift1; + + if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { + pext_mask = 0x0101010101010101ULL * mask; + } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { + pext_mask = 0x0001000100010001ULL * mask; + } else { + pext_mask = 0x0000000100000001ULL * mask; + } + + return { + mask, + pext_mask, + shift1, + shift2, + }; +} + +/** + * @brief Pack four 32-bit values with BMI2 extract instructions. + * + * @tparam BIT_WIDTH bit width per packed value. + * @param input SIMD register containing four quantized values. + * @return __m128i packed bitstream in the low bytes of the result. + */ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) +static inline auto mm_pack_epi32_bmi2(__m128i input) -> __m128i { + const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); + + if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 16) { + const __m128i packed = _mm_packs_epi32(input, _mm_setzero_si128()); + const u64 value = _pext_u64(_mm_extract_epi64(packed, 0), pext_mask); + + const __m128i result = _mm_set_epi64x(0, value); + return result; + } else { + alignas(16) u64 values[2]; + values[0] = _pext_u64(_mm_extract_epi64(input, 0), pext_mask); + + const u64 temp_combined = _pext_u64(_mm_extract_epi64(input, 1), pext_mask); + values[1] = temp_combined >> shift2; + values[0] |= (temp_combined << shift1); + + const __m128i result = _mm_set_epi64x(static_cast(values[1]), static_cast(values[0])); + return result; + } +} + +/** + * @brief Pack eight 32-bit values with BMI2 extract instructions. + * + * @tparam BIT_WIDTH bit width per packed value. + * @param input SIMD register containing eight quantized values. + * @return __m256i packed bitstream in the low bytes of the result. + */ +template + requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) +static inline auto mm256_pack_epi32_bmi2(__m256i input) -> __m256i { + const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); + + if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { + const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); + const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); + const __m256i packed8 = _mm256_packs_epi16(permuted, _mm256_setzero_si256()); + const u64 value = _pext_u64(_mm256_extract_epi64(packed8, 0), pext_mask); + + const __m256i result = _mm256_setr_epi64x(static_cast(value), 0, 0, 0); + return result; + } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { + const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); + alignas(16) i64 values[2] = {}; + values[0] = _pext_u64(_mm256_extract_epi64(packed16, 0), pext_mask); + + const u64 temp_combined = _pext_u64(_mm256_extract_epi64(packed16, 2), pext_mask); + values[1] = temp_combined >> shift2; + if constexpr (BIT_WIDTH != 16) { + values[0] |= static_cast(temp_combined << shift1); + } + + const __m256i result = _mm256_setr_epi64x(values[0], values[1], 0, 0); + return result; + } else { + constexpr u32 chunk_bits = BIT_WIDTH * 2; // bits extracted per 64-bit lane + static_assert(chunk_bits < 64); + + constexpr u64 chunk_mask = (chunk_bits == 64) ? ~u64{0} : ((u64{1} << chunk_bits) - 1); + + const u64 x0 = _pext_u64(_mm256_extract_epi64(input, 0), pext_mask) & chunk_mask; + const u64 x1 = _pext_u64(_mm256_extract_epi64(input, 1), pext_mask) & chunk_mask; + const u64 x2 = _pext_u64(_mm256_extract_epi64(input, 2), pext_mask) & chunk_mask; + const u64 x3 = _pext_u64(_mm256_extract_epi64(input, 3), pext_mask) & chunk_mask; + + u64 out0 = 0; + u64 out1 = 0; + u64 out2 = 0; + + auto append_bits = [&](u64 value, u32 bit_offset) { + const u32 word = bit_offset >> 6; // / 64 + const u32 off = bit_offset & 63; // % 64 + + if (word == 0) { + out0 |= value << off; + if (off + chunk_bits > 64) { + out1 |= value >> (64 - off); + } + } else if (word == 1) { + out1 |= value << off; + if (off + chunk_bits > 64) { + out2 |= value >> (64 - off); + } + } else { + out2 |= value << off; + } + }; + + append_bits(x0, 0 * chunk_bits); + append_bits(x1, 1 * chunk_bits); + append_bits(x2, 2 * chunk_bits); + append_bits(x3, 3 * chunk_bits); + + return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), static_cast(out2), 0); + } +} +} // namespace internal + +/** + * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * + * @param input pointer to the start of the input float values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + + std::memset(output, 0, BLOCK_SIZE); + + const __m256 scale_v = _mm256_set1_ps(scale); +#pragma GCC unroll 4 + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256 source = _mm256_loadu_ps(input); + const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); + const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); + std::memcpy(output, &packed, BIT_WIDTH); + input += 8; + output += BIT_WIDTH; + } + + if constexpr (remaining) { + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + const __m256 source = _mm256_maskload_ps(input, internal::mm256_tail_mask_epi32()); + const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); + const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); + const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); + std::memcpy(output, &packed, tail_bytes); + } + + return 0; +} + +/** + * @brief Compress a single block of double values using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @param input pointer to the start of the input double values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_block_bmi2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + + std::memset(output, 0, BLOCK_SIZE); + + const __m256d scale_v = _mm256_set1_pd(scale); +#pragma GCC unroll 4 + for (u32 iter = 0; iter < iterations_8; iter++) { + const __m256d source1 = _mm256_loadu_pd(input); + const __m256d source2 = _mm256_loadu_pd(input + 4); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed = internal::mm256_pack_epi32_bmi2(internal::mm256_clamp_signed_epi32(combined)); + std::memcpy(output, &packed, BIT_WIDTH); + input += 8; + output += BIT_WIDTH; + } + + if constexpr (remaining) { + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + constexpr u8 first_lanes = remaining < 4 ? remaining : 4; + constexpr u8 second_lanes = remaining > 4 ? remaining - 4 : 0; + + const __m256d source1 = _mm256_maskload_pd(input, internal::mm256_tail_mask_epi64()); + const __m256d source2 = [&] { + if constexpr (second_lanes > 0) { + return _mm256_maskload_pd(input + 4, internal::mm256_tail_mask_epi64()); + } else { + return _mm256_setzero_pd(); + } + }(); + const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); + const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); + __m256i combined = _mm256_castsi128_si256(quantized1); + combined = _mm256_inserti128_si256(combined, quantized2, 1); + const __m256i packed = internal::mm256_pack_epi32_bmi2(internal::mm256_clamp_signed_epi32(combined)); + std::memcpy(output, &packed, tail_bytes); + } + return 0; +} + +/** + * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * + * @param input pointer to the start of the input float values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @param blocks number of 512-bit blocks to compress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f32* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_bmi2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } + + return 0; +} + +/** + * @brief Compress multiple blocks of double values using AVX2 and BMI2 instructions. + * + * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). + * @param input pointer to the start of the input double values. + * @param scale scaling factor used during quantization. + * @param output pointer to the output buffer where compressed bytes will be stored. + * @param blocks number of blocks to compress. + * @return int status code (0 for success). + * + * @note This function requires AVX2 and BMI2 support. + */ +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) +int mm256_compress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const f64* block_input = input; + u8* block_output = output; + + for (u32 block = 0; block < blocks; block++) { + mm256_compress_block_bmi2(block_input, scale, block_output); + block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; + block_output += BLOCK_SIZE; + } + + return 0; +} +} // namespace pernix + +#endif // PERNIX_BMI2_COMPRESSION_H diff --git a/include/pernix/x86/bmi2/decompression.h b/include/pernix/x86/bmi2/bmi2_decompression.h similarity index 53% rename from include/pernix/x86/bmi2/decompression.h rename to include/pernix/x86/bmi2/bmi2_decompression.h index 443d673..823406a 100644 --- a/include/pernix/x86/bmi2/decompression.h +++ b/include/pernix/x86/bmi2/bmi2_decompression.h @@ -1,8 +1,8 @@ #ifndef PERNIX_BMI2_DECOMPRESSION_H #define PERNIX_BMI2_DECOMPRESSION_H -#include #include +#include #include #include @@ -18,7 +18,7 @@ namespace internal { * @param source register containing unpacked values. * @return __m128i sign-extended values. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __m128i mm_sign_extend32(__m128i source) { if constexpr (BIT_WIDTH == 1) { @@ -26,8 +26,8 @@ __m128i mm_sign_extend32(__m128i source) { return source; } - constexpr uint16_t shift = 32 - BIT_WIDTH; - source = _mm_slli_epi32(source, shift); + constexpr u16 shift = 32 - BIT_WIDTH; + source = _mm_slli_epi32(source, shift); return _mm_srai_epi32(source, shift); } @@ -38,7 +38,7 @@ __m128i mm_sign_extend32(__m128i source) { * @param source register containing unpacked values. * @return __m256i sign-extended values. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) __m256i mm256_sign_extend32(__m256i source) { if constexpr (BIT_WIDTH == 1) { @@ -46,8 +46,8 @@ __m256i mm256_sign_extend32(__m256i source) { return source; } - constexpr uint16_t shift = 32 - BIT_WIDTH; - source = _mm256_slli_epi32(source, shift); + constexpr u16 shift = 32 - BIT_WIDTH; + source = _mm256_slli_epi32(source, shift); return _mm256_srai_epi32(source, shift); } @@ -59,20 +59,20 @@ __m256i mm256_sign_extend32(__m256i source) { * @param input pointer to the packed input buffer. * @return __m128i unpacked values. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH > 0 && BIT_WIDTH <= 24) -__m128i mm_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { - constexpr uint32_t mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - constexpr std::size_t packed_bytes = (4 * BIT_WIDTH + 7) / 8; +__m128i mm_unpack_epi32_bmi2(const u8* __restrict__ input) { + constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + constexpr usize packed_bytes = (4 * BIT_WIDTH + 7) / 8; __m128i result; if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - constexpr uint64_t pdep_mask = 0x0101010101010101ULL * mask; + constexpr u64 pdep_mask = 0x0101010101010101ULL * mask; - uint32_t temp_value = 0; + u32 temp_value = 0; std::memcpy(&temp_value, input, packed_bytes); - const int32_t value = _pdep_u32(temp_value, static_cast(pdep_mask)); + const i32 value = _pdep_u32(temp_value, static_cast(pdep_mask)); const __m128i source = _mm_insert_epi32(_mm_setzero_si128(), value, 0); result = _mm_cvtepi8_epi32(source); @@ -80,24 +80,24 @@ __m128i mm_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { const __m128i source = _mm_loadu_si64(input); result = _mm_cvtepi16_epi32(source); } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { - constexpr uint64_t pdep_mask = 0x0001000100010001ULL * mask; + constexpr u64 pdep_mask = 0x0001000100010001ULL * mask; - uint64_t temp_value = 0; + u64 temp_value = 0; std::memcpy(&temp_value, input, packed_bytes); - const int64_t value = _pdep_u64(temp_value, pdep_mask); + const i64 value = _pdep_u64(temp_value, pdep_mask); const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); result = _mm_cvtepi16_epi32(source); } else { - constexpr uint64_t pdep_mask = 0x0000000100000001ULL * mask; - constexpr uint32_t shift1 = BIT_WIDTH * 2; - constexpr uint32_t shift2 = 64 - shift1; + constexpr u64 pdep_mask = 0x0000000100000001ULL * mask; + constexpr u32 shift1 = BIT_WIDTH * 2; + constexpr u32 shift2 = 64 - shift1; - alignas(16) uint64_t temp_values[2]{}; + alignas(16) u64 temp_values[2]{}; std::memcpy(temp_values, input, packed_bytes); - alignas(16) int64_t values[2]; + alignas(16) i64 values[2]; values[0] = _pdep_u64(temp_values[0], pdep_mask); values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); @@ -118,20 +118,24 @@ __m128i mm_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { * @param input pointer to the packed input buffer. * @return __m256i unpacked values. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) -__m256i mm256_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { - constexpr uint32_t mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - constexpr std::size_t packed_bytes = BIT_WIDTH; +__m256i mm256_unpack_epi32_bmi2(const __m256i packed_source) { + constexpr u32 mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; + constexpr usize packed_bytes = BIT_WIDTH; + + alignas(32) u64 packed_words[4]; + _mm256_store_si256(reinterpret_cast<__m256i*>(packed_words), packed_source); + const auto* input = reinterpret_cast(packed_words); __m256i result; if constexpr (BIT_WIDTH >= 1 && BIT_WIDTH <= 8) { - constexpr uint64_t pdep_mask = 0x0101010101010101ULL * mask; + constexpr u64 pdep_mask = 0x0101010101010101ULL * mask; - uint64_t temp_value = 0; + u64 temp_value = 0; std::memcpy(&temp_value, input, packed_bytes); - const int64_t value = _pdep_u64(temp_value, pdep_mask); + const i64 value = _pdep_u64(temp_value, pdep_mask); const __m128i source = _mm_insert_epi64(_mm_setzero_si128(), value, 0); result = _mm256_cvtepi8_epi32(source); @@ -139,49 +143,49 @@ __m256i mm256_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { const __m128i source = _mm_loadu_si128(reinterpret_cast(input)); result = _mm256_cvtepi16_epi32(source); } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { - constexpr uint64_t pdep_mask = 0x0001000100010001ULL * mask; - constexpr uint64_t shift1 = BIT_WIDTH * 4; - constexpr uint64_t shift2 = 64 - shift1; + constexpr u64 pdep_mask = 0x0001000100010001ULL * mask; + constexpr u64 shift1 = BIT_WIDTH * 4; + constexpr u64 shift2 = 64 - shift1; - alignas(16) uint64_t temp_values[2]{}; + alignas(16) u64 temp_values[2]{}; std::memcpy(temp_values, input, packed_bytes); - alignas(16) int64_t values[2]; + alignas(16) i64 values[2]; values[0] = _pdep_u64(temp_values[0], pdep_mask); values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); const __m128i source = _mm_set_epi64x(values[1], values[0]); result = _mm256_cvtepi16_epi32(source); } else { - constexpr uint64_t pdep_mask = 0x0000000100000001ULL * mask; - constexpr uint32_t shift1 = BIT_WIDTH * 2; - constexpr uint32_t shift2 = 64 - shift1; + constexpr u64 pdep_mask = 0x0000000100000001ULL * mask; + constexpr u32 shift1 = BIT_WIDTH * 2; + constexpr u32 shift2 = 64 - shift1; - alignas(16) uint64_t temp_values[4]{}; + alignas(16) u64 temp_values[4]{}; std::memcpy(temp_values, input, packed_bytes); if constexpr ((BIT_WIDTH % 2) == 0) { std::memcpy(temp_values + 2, input + BIT_WIDTH / 2, packed_bytes - BIT_WIDTH / 2); } else { - constexpr uint32_t second_group_bit_offset = BIT_WIDTH * 4; - constexpr uint32_t second_group_byte_offset = second_group_bit_offset / 8; - constexpr uint32_t second_group_shift = second_group_bit_offset % 8; + constexpr u32 second_group_bit_offset = BIT_WIDTH * 4; + constexpr u32 second_group_byte_offset = second_group_bit_offset / 8; + constexpr u32 second_group_shift = second_group_bit_offset % 8; - alignas(16) uint64_t raw_values[2]{}; + alignas(16) u64 raw_values[2]{}; std::memcpy(raw_values, input + second_group_byte_offset, packed_bytes - second_group_byte_offset); temp_values[2] = (raw_values[0] >> second_group_shift) | (raw_values[1] << (64 - second_group_shift)); temp_values[3] = raw_values[1] >> second_group_shift; } - alignas(16) uint64_t values[4]; + alignas(16) u64 values[4]; values[0] = _pdep_u64((temp_values[0]), pdep_mask); values[1] = _pdep_u64((temp_values[0] >> shift1) | (temp_values[1] << shift2), pdep_mask); values[2] = _pdep_u64((temp_values[2]), pdep_mask); values[3] = _pdep_u64((temp_values[2] >> shift1) | (temp_values[3] << shift2), pdep_mask); - result = _mm256_set_epi64x(static_cast(values[3]), static_cast(values[2]), static_cast(values[1]), - static_cast(values[0])); + result = _mm256_set_epi64x(static_cast(values[3]), static_cast(values[2]), static_cast(values[1]), + static_cast(values[0])); } if constexpr (SIGN_VALUES) { @@ -189,7 +193,15 @@ __m256i mm256_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { } return result; } -} // namespace internal + +template + requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) +__m256i mm256_unpack_epi32_bmi2(const u8* __restrict__ input) { + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, BIT_WIDTH); + return mm256_unpack_epi32_bmi2(source); +} +} // namespace internal /** * @brief Decompress a single 512\-bit block using AVX2 and BMI2 instructions. @@ -205,28 +217,34 @@ __m256i mm256_unpack_epi32_bmi2(const uint8_t* __restrict__ input) { * * @note This function requires AVX2 and BMI2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_bmi2(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; +int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; const __m256 scale_v = _mm256_set1_ps(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { + for (u32 iter = 0; iter < iterations_8; iter++) { const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); _mm256_storeu_ps(output, dequantized); - input += BIT_WIDTH; + input += BIT_WIDTH; output += 8; } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (uint32_t i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi32(tail_values[i], scale); - } + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, tail_bytes); + + const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(source); + const __m256 dequantized = internal::mm256_dequantize_epi32(unpacked, scale_v); + std::memcpy(output, &dequantized, remaining * sizeof(f32)); } return 0; @@ -246,15 +264,18 @@ int mm256_decompress_block_bmi2(const uint8_t* __restrict__ input, const float_t * * @note This function requires AVX2 and BMI2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_block_bmi2(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; - const __m256d scale_v = _mm256_set1_pd(scale); +int mm256_decompress_block_bmi2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + constexpr u32 elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; + constexpr u32 iterations_8 = elements_per_block / 8; + constexpr u8 remaining = elements_per_block - iterations_8 * 8; + const __m256d scale_v = _mm256_set1_pd(scale); #pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { + for (u32 iter = 0; iter < iterations_8; iter++) { const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(input); const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); @@ -265,14 +286,26 @@ int mm256_decompress_block_bmi2(const uint8_t* __restrict__ input, const double_ _mm256_storeu_pd(output, dequantized1); _mm256_storeu_pd(output + 4, dequantized2); - input += BIT_WIDTH; + input += BIT_WIDTH; output += 8; } if constexpr (remaining > 0) { - const std::vector tail_values = internal::unpack_epi32_fallback(input, remaining); - for (uint32_t i = 0; i < remaining; i++) { - output[i] = internal::dequantize_epi64(tail_values[i], scale); + constexpr u32 tail_bytes = (BIT_WIDTH * remaining + 7) / 8; + __m256i source = _mm256_setzero_si256(); + std::memcpy(&source, input, tail_bytes); + + const __m256i unpacked = internal::mm256_unpack_epi32_bmi2(source); + const __m256i extend1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(unpacked)); + const __m256i extend2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(unpacked, 1)); + + const __m256d dequantized1 = internal::mm256_dequantize_epi64_pd(extend1, scale_v); + const __m256d dequantized2 = internal::mm256_dequantize_epi64_pd(extend2, scale_v); + + constexpr u32 first_elements = remaining < 4 ? remaining : 4; + std::memcpy(output, &dequantized1, first_elements * sizeof(f64)); + if constexpr (remaining > 4) { + std::memcpy(output + 4, &dequantized2, (remaining - 4) * sizeof(f64)); } } @@ -294,16 +327,18 @@ int mm256_decompress_block_bmi2(const uint8_t* __restrict__ input, const double_ * * @note This function requires AVX2 and BMI2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_bmi2(const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - float_t* block_output = output; +int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f32 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); + + const u8* block_input = input; + f32* block_output = output; - for (uint32_t block = 0; block < blocks; block++) { + for (u32 block = 0; block < blocks; block++) { mm256_decompress_block_bmi2(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } @@ -325,87 +360,23 @@ int mm256_decompress_blocks_bmi2(const uint8_t* __restrict__ input, const float_ * * @note This function requires AVX2 and BMI2 support. */ -template +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_decompress_blocks_bmi2(const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - const uint8_t* block_input = input; - double_t* block_output = output; +int mm256_decompress_blocks_bmi2(const void* __restrict__ input_ptr, const f64 scale, void* __restrict__ output_ptr, const u32 blocks) { + const auto* input = static_cast(input_ptr); + auto* output = static_cast(output_ptr); - for (uint32_t block = 0; block < blocks; block++) { + const u8* block_input = input; + f64* block_output = output; + + for (u32 block = 0; block < blocks; block++) { mm256_decompress_block_bmi2(block_input, scale, block_output); - block_input += BLOCK_SIZE; + block_input += BLOCK_SIZE; block_output += (BLOCK_SIZE * 8) / BIT_WIDTH; } return 0; } -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Decompress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_decompress_block_bmi2(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output); - -/** - * @brief Decompress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the compressed block. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_decompress_block_f64_bmi2(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output); - -/** - * @brief Decompress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_decompress_blocks_bmi2(uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Decompress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 16). - * @param input pointer to the start of the compressed data. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where decompressed float values will be stored. - * @param blocks number of 512-bit blocks to decompress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_decompress_blocks_f64_bmi2(uint8_t bit_width, const uint8_t* __restrict__ input, double_t scale, double_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif +} // namespace pernix #endif // PERNIX_BMI2_DECOMPRESSION_H diff --git a/include/pernix/x86/bmi2/compression.h b/include/pernix/x86/bmi2/compression.h deleted file mode 100644 index ca4b976..0000000 --- a/include/pernix/x86/bmi2/compression.h +++ /dev/null @@ -1,371 +0,0 @@ -#ifndef PERNIX_BMI2_COMPRESSION_H -#define PERNIX_BMI2_COMPRESSION_H - -#include -#include -#include - -#include -#include -#include - -namespace pernix { -namespace internal { -/** - * @brief Build the masks and shift constants used by the BMI2 packers. - * - * @tparam BIT_WIDTH bit width per packed value. - * @return std::tuple mask tuple used by the BMI2 helpers. - */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) -static constexpr std::tuple pack_avx2_bmi2_constants() { - uint32_t mask = BIT_WIDTH == 32 ? std::numeric_limits::max() : (1ULL << BIT_WIDTH) - 1U; - uint64_t pext_mask; - uint16_t shift1 = BIT_WIDTH * 4; - uint16_t shift2 = 64 - shift1; - - if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { - pext_mask = 0x0101010101010101ULL * mask; - } else if constexpr (BIT_WIDTH > 8 && BIT_WIDTH <= 16) { - pext_mask = 0x0001000100010001ULL * mask; - } else { - pext_mask = 0x0000000100000001ULL * mask; - } - - return { - mask, - pext_mask, - shift1, - shift2, - }; -} - -/** - * @brief Pack four 32-bit values with BMI2 extract instructions. - * - * @tparam BIT_WIDTH bit width per packed value. - * @param input SIMD register containing four quantized values. - * @return __m128i packed bitstream in the low bytes of the result. - */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 32) -static inline auto mm_pack_epi32_bmi2(const __m128i& input) -> __m128i { - const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); - - if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 16) { - const __m128i packed = _mm_packs_epi32(input, _mm_setzero_si128()); - const uint64_t value = _pext_u64(_mm_extract_epi64(packed, 0), pext_mask); - - const __m128i result = _mm_set_epi64x(0, value); - return result; - } else { - alignas(16) uint64_t values[2]; - values[0] = _pext_u64(_mm_extract_epi64(input, 0), pext_mask); - - const uint64_t temp_combined = _pext_u64(_mm_extract_epi64(input, 1), pext_mask); - values[1] = temp_combined >> shift2; - values[0] |= (temp_combined << shift1); - - const __m128i result = _mm_set_epi64x(static_cast(values[1]), static_cast(values[0])); - return result; - } -} - -/** - * @brief Pack eight 32-bit values with BMI2 extract instructions. - * - * @tparam BIT_WIDTH bit width per packed value. - * @param input SIMD register containing eight quantized values. - * @return __m256i packed bitstream in the low bytes of the result. - */ -template - requires(BIT_WIDTH > 0 && BIT_WIDTH <= 24) -static inline auto mm256_pack_epi32_bmi2(const __m256i& input) -> __m256i { - const auto [mask, pext_mask, shift1, shift2] = pack_avx2_bmi2_constants(); - - if constexpr (BIT_WIDTH > 0 && BIT_WIDTH <= 8) { - const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); - const __m256i permuted = _mm256_permute4x64_epi64(packed16, _MM_SHUFFLE(3, 1, 2, 0)); - const __m256i packed8 = _mm256_packs_epi16(permuted, _mm256_setzero_si256()); - const uint64_t value = _pext_u64(_mm256_extract_epi64(packed8, 0), pext_mask); - - const __m256i result = _mm256_setr_epi64x(static_cast(value), 0, 0, 0); - return result; - } else if constexpr (BIT_WIDTH >= 9 && BIT_WIDTH <= 16) { - const __m256i packed16 = _mm256_packs_epi32(input, _mm256_setzero_si256()); - alignas(16) int64_t values[2] = {}; - values[0] = _pext_u64(_mm256_extract_epi64(packed16, 0), pext_mask); - - const uint64_t temp_combined = _pext_u64(_mm256_extract_epi64(packed16, 2), pext_mask); - values[1] = temp_combined >> shift2; - if constexpr (BIT_WIDTH != 16) { - values[0] |= static_cast(temp_combined << shift1); - } - - const __m256i result = _mm256_setr_epi64x(values[0], values[1], 0, 0); - return result; - } else { - constexpr uint32_t chunk_bits = BIT_WIDTH * 2; // bits extracted per 64-bit lane - static_assert(chunk_bits < 64); - - constexpr uint64_t chunk_mask = (chunk_bits == 64) ? ~uint64_t{0} : ((uint64_t{1} << chunk_bits) - 1); - - const uint64_t x0 = _pext_u64(_mm256_extract_epi64(input, 0), pext_mask) & chunk_mask; - const uint64_t x1 = _pext_u64(_mm256_extract_epi64(input, 1), pext_mask) & chunk_mask; - const uint64_t x2 = _pext_u64(_mm256_extract_epi64(input, 2), pext_mask) & chunk_mask; - const uint64_t x3 = _pext_u64(_mm256_extract_epi64(input, 3), pext_mask) & chunk_mask; - - uint64_t out0 = 0; - uint64_t out1 = 0; - uint64_t out2 = 0; - - auto append_bits = [&](uint64_t value, uint32_t bit_offset) { - const uint32_t word = bit_offset >> 6; // / 64 - const uint32_t off = bit_offset & 63; // % 64 - - if (word == 0) { - out0 |= value << off; - if (off + chunk_bits > 64) { - out1 |= value >> (64 - off); - } - } else if (word == 1) { - out1 |= value << off; - if (off + chunk_bits > 64) { - out2 |= value >> (64 - off); - } - } else { - out2 |= value << off; - } - }; - - append_bits(x0, 0 * chunk_bits); - append_bits(x1, 1 * chunk_bits); - append_bits(x2, 2 * chunk_bits); - append_bits(x3, 3 * chunk_bits); - - return _mm256_setr_epi64x(static_cast(out0), static_cast(out1), static_cast(out2), 0); - } -} -} // namespace internal - -/** - * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_bmi2(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; - - std::memset(output, 0, BLOCK_SIZE); - - const __m256 scale_v = _mm256_set1_ps(scale); -#pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256 source = _mm256_loadu_ps(input); - const __m256i quantized = internal::mm256_quantize_ps_epi32(source, scale_v); - const __m256i packed_input = internal::mm256_clamp_signed_epi32(quantized); - const __m256i packed = internal::mm256_pack_epi32_bmi2(packed_input); - std::memcpy(output, &packed, BIT_WIDTH); - input += 8; - output += BIT_WIDTH; - } - - if constexpr (remaining) { - std::vector block_values(remaining); -#pragma GCC unroll 8 - for (uint32_t i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized(internal::quantize_ps_epi32(input[i], scale))); - } - - internal::pack_epi32_fallback(block_values, output); - } - - return 0; -} - -/** - * @brief Compress a single block of double values using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_block_bmi2(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - constexpr uint32_t elements_per_block = (BLOCK_SIZE * 8) / BIT_WIDTH; - constexpr uint32_t iterations_8 = elements_per_block / 8; - constexpr uint8_t remaining = elements_per_block - iterations_8 * 8; - - std::memset(output, 0, BLOCK_SIZE); - - const __m256d scale_v = _mm256_set1_pd(scale); -#pragma GCC unroll 4 - for (uint32_t iter = 0; iter < iterations_8; iter++) { - const __m256d source1 = _mm256_loadu_pd(input); - const __m256d source2 = _mm256_loadu_pd(input + 4); - const __m128i quantized1 = internal::mm256_quantize_pd_epi32(source1, scale_v); - const __m128i quantized2 = internal::mm256_quantize_pd_epi32(source2, scale_v); - __m256i combined = _mm256_castsi128_si256(quantized1); - combined = _mm256_inserti128_si256(combined, quantized2, 1); - const __m256i packed = internal::mm256_pack_epi32_bmi2(internal::mm256_clamp_signed_epi32(combined)); - std::memcpy(output, &packed, BIT_WIDTH); - input += 8; - output += BIT_WIDTH; - } - - if constexpr (remaining) { - std::vector block_values(remaining); -#pragma GCC unroll 8 - for (uint32_t i = 0; i < remaining; i++) { - block_values[i] = - static_cast(internal::clamp_signed_quantized(internal::quantize_pd_epi64(input[i], scale))); - } - - internal::pack_epi32_fallback(block_values, output); - } - return 0; -} - -/** - * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_bmi2(const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - const float_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_compress_block_bmi2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - - return 0; -} - -/** - * @brief Compress multiple blocks of double values using AVX2 and BMI2 instructions. - * - * @tparam BIT_WIDTH bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -template - requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && (BLOCK_SIZE % 32 == 0) -int mm256_compress_blocks_bmi2(const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - const double_t* block_input = input; - uint8_t* block_output = output; - - for (uint32_t block = 0; block < blocks; block++) { - mm256_compress_block_bmi2(block_input, scale, block_output); - block_input += (BLOCK_SIZE * 8) / BIT_WIDTH; - block_output += BLOCK_SIZE; - } - - return 0; -} -} // namespace pernix - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -/** - * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_block_bmi2(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress a single 512-bit block using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_block_f64_bmi2(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output); - -/** - * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input float values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_blocks_bmi2(uint8_t bit_width, const float_t* __restrict__ input, float_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -/** - * @brief Compress multiple 512-bit blocks using AVX2 and BMI2 instructions. - * - * @param bit_width bit width per value in the packed representation (1 to 24). - * @param input pointer to the start of the input double values. - * @param scale scaling factor used during quantization. - * @param output pointer to the output buffer where compressed bytes will be stored. - * @param blocks number of 512-bit blocks to compress. - * @return int status code (0 for success). - * - * @note This function requires AVX2 and BMI2 support. - */ -int mm256_compress_blocks_f64_bmi2(uint8_t bit_width, const double_t* __restrict__ input, double_t scale, uint8_t* __restrict__ output, - uint32_t blocks); - -#ifdef __cplusplus -} -} // namespace pernix -#endif - -#endif // PERNIX_BMI2_COMPRESSION_H diff --git a/include/pernix/x86/utils.h b/include/pernix/x86/utils.h new file mode 100644 index 0000000..3401b03 --- /dev/null +++ b/include/pernix/x86/utils.h @@ -0,0 +1,14 @@ +#ifndef PERNIX_X86_UTILS_H +#define PERNIX_X86_UTILS_H + +#include + +namespace pernix::x86::internal { +static constexpr u32 tail_bytes(const u8 bit_width, const u32 remaining_elements) { + const u32 tail_bits = remaining_elements * bit_width; + const u32 tail_bytes = (tail_bits + 7u) / 8u; + return tail_bytes; +} +} // namespace pernix::x86::internal + +#endif // PERNIX_X86_UTILS_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b7e8f23..6567f8b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,67 +1,255 @@ include(GNUInstallDirs) +include(CMakePackageConfigHelpers) -file(GLOB_RECURSE - PERNIX_COMMON_SOURCES - ./fallback/*.cpp - ./pernix.cpp - ${PROJECT_SOURCE_DIR}/include/pernix/*.h -) - -set(PERNIX_SOURCES ${PERNIX_COMMON_SOURCES}) +add_library(pernix) +add_library(pernix::pernix ALIAS pernix) +add_library(pernix_header_only INTERFACE) +add_library(pernix::pernix_header_only ALIAS pernix_header_only) -set(PERNIX_TARGET_IS_X86 OFF) -if (PERNIX_USE_SIMDE OR CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|i[3-6]86|i686)$") - set(PERNIX_TARGET_IS_X86 ON) -endif () +set(PERNIX_GENERIC_SOURCE_FILES + pernix.cpp + dispatch/select.cpp + fallback/fallback_compression.cpp + fallback/fallback_decompression.cpp +) if (PERNIX_TARGET_IS_X86) - file(GLOB_RECURSE - PERNIX_X86_SOURCES - ./x86/*.cpp - ${PROJECT_SOURCE_DIR}/include/pernix/x86/*.h - ) - list(APPEND PERNIX_SOURCES ${PERNIX_X86_SOURCES}) + list(APPEND PERNIX_GENERIC_SOURCE_FILES dispatch/cpu_features_x86.cpp) +elseif (PERNIX_TARGET_IS_ARM64) + list(APPEND PERNIX_GENERIC_SOURCE_FILES dispatch/cpu_features_arm.cpp) +else () + list(APPEND PERNIX_GENERIC_SOURCE_FILES dispatch/cpu_features_x86.cpp) endif () -add_library(pernix SHARED ${PERNIX_SOURCES}) +target_sources(pernix PRIVATE ${PERNIX_GENERIC_SOURCE_FILES}) +target_compile_features(pernix PUBLIC cxx_std_20) +target_compile_features(pernix_header_only INTERFACE cxx_std_20) +target_compile_options(pernix PRIVATE ${PERNIX_PRIVATE_COMPILE_OPTIONS}) + set_target_properties(pernix PROPERTIES OUTPUT_NAME "pernix" VERSION ${NORMALIZED_VERSION} + LINKER_LANGUAGE CXX +) + +target_include_directories(pernix + PUBLIC + $ + $ ) -target_include_directories(pernix PUBLIC + +target_include_directories(pernix_header_only + INTERFACE $ + $ ) +target_compile_definitions(pernix PRIVATE PERNIX_BUILD_LIB=1) +if (BUILD_SHARED_LIBS) + target_compile_definitions(pernix PUBLIC PERNIX_SHARED=1) +endif () + +function(pernix_set_source_options) + set(options) + set(one_value_args) + set(multi_value_args SOURCES OPTIONS) + cmake_parse_arguments(PERNIX_SOURCE_OPTIONS "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + foreach (PERNIX_SOURCE_FILE IN LISTS PERNIX_SOURCE_OPTIONS_SOURCES) + set_source_files_properties("${PERNIX_SOURCE_FILE}" PROPERTIES + COMPILE_OPTIONS "${PERNIX_SOURCE_OPTIONS_OPTIONS}" + ) + endforeach () +endfunction() + +if (PERNIX_ENABLE_LTO) + set_target_properties(pernix PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) +endif () + if (PERNIX_USE_SIMDE) - target_link_libraries(pernix PUBLIC simde::simde) target_compile_definitions(pernix PUBLIC PERNIX_USE_SIMDE=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_USE_SIMDE=1) endif () -set_target_properties(pernix PROPERTIES LINKER_LANGUAGE CXX) +set(PERNIX_BUILD_FALLBACK_STDPAR OFF) +if (NOT PERNIX_ENABLE_FALLBACK_STDPAR STREQUAL "OFF") + set(PERNIX_BUILD_FALLBACK_STDPAR ON) + find_package(TBB QUIET) -configure_file( - "${PROJECT_SOURCE_DIR}/cmake/pernix.pc.in" "${PROJECT_BINARY_DIR}/pernix.pc" @ONLY -) + target_sources(pernix PRIVATE + fallback/stdpar_compression.cpp + fallback/stdpar_decompression.cpp + ) + target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_STDPAR=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_STDPAR=1) + + if (TARGET TBB::tbb) + target_compile_definitions(pernix PUBLIC PERNIX_STDPAR_USE_PARALLEL_POLICY=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_STDPAR_USE_PARALLEL_POLICY=1) + target_link_libraries(pernix PUBLIC TBB::tbb) + target_link_libraries(pernix_header_only INTERFACE TBB::tbb) + message(STATUS "PERNIX fallback stdpar backend enabled with parallel policy (TBB found)") + else () + message(STATUS "PERNIX fallback stdpar backend enabled without TBB; stdpar will execute sequentially") + endif () +endif () + +if (PERNIX_BUILD_FALLBACK_SIMD) + target_sources(pernix PRIVATE + fallback/simd_compression.cpp + fallback/simd_decompression.cpp + ) + target_compile_features(pernix PUBLIC cxx_std_26) + target_compile_features(pernix_header_only INTERFACE cxx_std_26) + target_compile_definitions(pernix PUBLIC PERNIX_BUILD_FALLBACK_SIMD=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_BUILD_FALLBACK_SIMD=1) +endif () + +if (PERNIX_ENABLE_X86_BMI2 AND PERNIX_TARGET_IS_X86) + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_BMI2=1) + target_sources(pernix PRIVATE + x86/bmi2/bmi2_compression.cpp + x86/bmi2/bmi2_decompression.cpp + ) + if (MSVC) + pernix_set_source_options(SOURCES + x86/bmi2/bmi2_compression.cpp + x86/bmi2/bmi2_decompression.cpp + OPTIONS /arch:AVX2) + else () + pernix_set_source_options(SOURCES + x86/bmi2/bmi2_compression.cpp + x86/bmi2/bmi2_decompression.cpp + OPTIONS -mavx2 -mbmi -mbmi2) + endif () +endif () + +if (PERNIX_ENABLE_X86_AVX2 AND PERNIX_TARGET_IS_X86) + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_AVX2=1) + target_sources(pernix PRIVATE + x86/avx2/avx2_compression.cpp + x86/avx2/avx2_decompression.cpp + ) + if (MSVC) + pernix_set_source_options(SOURCES + x86/avx2/avx2_compression.cpp + x86/avx2/avx2_decompression.cpp + OPTIONS /arch:AVX2) + else () + pernix_set_source_options(SOURCES + x86/avx2/avx2_compression.cpp + x86/avx2/avx2_decompression.cpp + OPTIONS -mavx2) + endif () +endif () + +if (PERNIX_ENABLE_X86_AVX512VBMI AND PERNIX_TARGET_IS_X86) + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_X86_AVX512_VBMI=1) + target_sources(pernix PRIVATE + x86/avx512vbmi/avx512vbmi_compression.cpp + x86/avx512vbmi/avx512vbmi_decompression.cpp + ) + if (MSVC) + pernix_set_source_options(SOURCES + x86/avx512vbmi/avx512vbmi_compression.cpp + x86/avx512vbmi/avx512vbmi_decompression.cpp + OPTIONS /arch:AVX512) + else () + pernix_set_source_options(SOURCES + x86/avx512vbmi/avx512vbmi_compression.cpp + x86/avx512vbmi/avx512vbmi_decompression.cpp + OPTIONS -mavx -mbmi -mbmi2 -mavx2 -mavx512f -mavx512bw -mavx512vl -mavx512dq -mavx512cd -mavx512vbmi) + endif () +endif () + +if (PERNIX_ENABLE_ARM64_NEON AND PERNIX_TARGET_IS_ARM64) + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_ARM64_NEON=1) + target_sources(pernix PRIVATE + arm64/neon/compression.cpp + arm64/neon/decompression.cpp + ) + pernix_set_source_options(SOURCES + arm64/neon/compression.cpp + arm64/neon/decompression.cpp + OPTIONS -march=armv8-a+simd) +endif () + +if (PERNIX_ENABLE_ARM64_SVE2 AND PERNIX_TARGET_IS_ARM64) + target_compile_definitions(pernix PRIVATE PERNIX_BUILD_ARM64_SVE2=1) + target_sources(pernix PRIVATE + arm64/sve2/compression.cpp + arm64/sve2/decompression.cpp + ) + pernix_set_source_options(SOURCES + arm64/sve2/compression.cpp + arm64/sve2/decompression.cpp + OPTIONS -march=armv8.2-a+simd+sve2) +endif () + +if (PERNIX_DISABLE_BMI2) + target_compile_definitions(pernix PUBLIC PERNIX_DISABLE_BMI2=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_DISABLE_BMI2=1) +endif () + +if (PERNIX_DISABLE_AVX2) + target_compile_definitions(pernix PUBLIC PERNIX_DISABLE_AVX2=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_DISABLE_AVX2=1) +endif () + +if (PERNIX_DISABLE_AVX512) + target_compile_definitions(pernix PUBLIC PERNIX_DISABLE_AVX512=1) + target_compile_definitions(pernix_header_only INTERFACE PERNIX_DISABLE_AVX512=1) +endif () + +set(PERNIX_PC_REQUIRES "") + +configure_file("${PROJECT_SOURCE_DIR}/cmake/pernix.pc.in" "${PROJECT_BINARY_DIR}/pernix.pc" @ONLY) if (PERNIX_ENABLE_INSTALL) - install(TARGETS pernix + install(TARGETS pernix pernix_header_only + EXPORT pernixTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) - install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/pernix ${PROJECT_BINARY_DIR}/include/pernix + install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/pernix" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - FILES_MATCHING PATTERN "*.*h" + FILES_MATCHING + PATTERN "*.h" + PATTERN "*.hpp" ) + if (PERNIX_USE_SIMDE AND PERNIX_BUNDLE_SIMDE_FOR_INSTALL) + install(DIRECTORY "${simde_SOURCE_DIR}/simde" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + FILES_MATCHING PATTERN "*.h" + ) + endif () + + configure_package_config_file( + "${PROJECT_SOURCE_DIR}/cmake/pernixConfig.cmake.in" + "${PROJECT_BINARY_DIR}/pernixConfig.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/pernix" + ) + write_basic_package_version_file( + "${PROJECT_BINARY_DIR}/pernixConfigVersion.cmake" + VERSION ${NORMALIZED_VERSION} + COMPATIBILITY SameMajorVersion + ) install( - FILES ${PROJECT_BINARY_DIR}/pernix.pc - DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" + FILES + "${PROJECT_BINARY_DIR}/pernixConfig.cmake" + "${PROJECT_BINARY_DIR}/pernixConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/pernix" ) - - add_custom_target(uninstall COMMAND xargs rm -vf < ${PROJECT_BINARY_DIR}/install_manifest.txt) + install( + EXPORT pernixTargets + FILE pernixTargets.cmake + NAMESPACE pernix:: + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/pernix" + ) + install(FILES "${PROJECT_BINARY_DIR}/pernix.pc" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") endif () if (PERNIX_ENABLE_DOXYGEN) @@ -78,16 +266,14 @@ if (PERNIX_ENABLE_DOXYGEN) set(DOXYGEN_SHOW_INCLUDE_FILES YES) set(DOXYGEN_BINARY_TOC YES) set(DOXYGEN_TOC_EXPAND YES) - # set(DOXYGEN_USE_MDFILE_AS_MAINPAGE "index.md") doxygen_add_docs(pernix_doxygen - # docs include src ALL WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMENT "Generating documentation with Doxygen." ) - if (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS) + if (PERNIX_ENABLE_INSTALL AND PERNIX_INSTALL_DOCS) install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html/" DESTINATION ${CMAKE_INSTALL_DOCDIR}) endif () diff --git a/src/arm64/neon/compression.cpp b/src/arm64/neon/compression.cpp new file mode 100644 index 0000000..5278adb --- /dev/null +++ b/src/arm64/neon/compression.cpp @@ -0,0 +1,28 @@ +#include +#include + +namespace pernix::internal { +Kernel select_neon_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} + +Kernel select_neon_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} + +Kernel select_neon_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} + +Kernel select_neon_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"neon", nullptr}; +} +} // namespace pernix::internal diff --git a/src/arm64/neon/decompression.cpp b/src/arm64/neon/decompression.cpp new file mode 100644 index 0000000..3acb4bb --- /dev/null +++ b/src/arm64/neon/decompression.cpp @@ -0,0 +1,216 @@ +#include +#include + +using pernix::arm64::neon::neon_decompress_block; +using pernix::arm64::neon::neon_decompress_blocks; + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_block); \ + return Kernel("neon", &neon_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("neon", &neon_decompress_blocks); \ + return Kernel("neon", &neon_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"neon", nullptr}; \ + } \ + break + +Kernel select_neon_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"neon", nullptr}; + } +} + +Kernel select_neon_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"neon", nullptr}; + } +} + +Kernel select_neon_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"neon", nullptr}; + } +} + +Kernel select_neon_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"neon", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal diff --git a/src/arm64/sve2/compression.cpp b/src/arm64/sve2/compression.cpp new file mode 100644 index 0000000..262254d --- /dev/null +++ b/src/arm64/sve2/compression.cpp @@ -0,0 +1,28 @@ +#include +#include + +namespace pernix::internal { +Kernel select_sve2_compress_block_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} + +Kernel select_sve2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} + +Kernel select_sve2_compress_block_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} + +Kernel select_sve2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + (void)bit_width; + (void)block_size; + return {"sve2", nullptr}; +} +} // namespace pernix::internal diff --git a/src/arm64/sve2/decompression.cpp b/src/arm64/sve2/decompression.cpp new file mode 100644 index 0000000..418a2ea --- /dev/null +++ b/src/arm64/sve2/decompression.cpp @@ -0,0 +1,216 @@ +#include +#include + +using pernix::arm64::sve2::sve2_decompress_block; +using pernix::arm64::sve2::sve2_decompress_blocks; + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_block); \ + return Kernel("sve2", &sve2_decompress_block) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("sve2", &sve2_decompress_blocks); \ + return Kernel("sve2", &sve2_decompress_blocks) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"sve2", nullptr}; \ + } \ + break + +Kernel select_sve2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"sve2", nullptr}; + } +} + +Kernel select_sve2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"sve2", nullptr}; + } +} + +Kernel select_sve2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"sve2", nullptr}; + } +} + +Kernel select_sve2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"sve2", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal diff --git a/src/dispatch/cpu_features_arm.cpp b/src/dispatch/cpu_features_arm.cpp new file mode 100644 index 0000000..516041e --- /dev/null +++ b/src/dispatch/cpu_features_arm.cpp @@ -0,0 +1,39 @@ +#include + +#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) +#include +#ifndef HWCAP_SVE +#define HWCAP_SVE (1 << 22) +#endif +#ifndef HWCAP2_SVE2 +#define HWCAP2_SVE2 (1 << 1) +#endif +#endif + +namespace pernix::internal { +CpuFeatures detect_cpu_features() { + CpuFeatures features{}; + + // neon +#if defined(__aarch64__) || defined(_M_ARM64) + features.neon = true; +#elif defined(__ARM_NEON) || defined(__ARM_NEON__) + features.neon = true; +#endif + + // sve / sve2 — runtime detection via getauxval on Linux +#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64)) + unsigned long hwcap = getauxval(AT_HWCAP); + unsigned long hwcap2 = getauxval(AT_HWCAP2); + features.sve = (hwcap & HWCAP_SVE) != 0; + features.sve2 = (hwcap2 & HWCAP2_SVE2) != 0; +#endif + + return features; +} + +CpuFeatures get_cached_cpu_features() { + static const CpuFeatures features = detect_cpu_features(); + return features; +} +} // namespace pernix::internal diff --git a/src/dispatch/cpu_features_x86.cpp b/src/dispatch/cpu_features_x86.cpp new file mode 100644 index 0000000..94496e2 --- /dev/null +++ b/src/dispatch/cpu_features_x86.cpp @@ -0,0 +1,102 @@ +#include +#include + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386) || defined(_M_IX86) +#if defined(_MSC_VER) +#include +#else +#include +#include +#endif + +namespace pernix::internal { +namespace { +#if defined(_MSC_VER) + +void cpuid(int out[4], int leaf, int subleaf) { + __cpuidex(out, leaf, subleaf); +} + +u64 xgetbv(unsigned int index) { + return _xgetbv(index); +} + +#else + +void cpuid(int out[4], int leaf, int subleaf) { + __cpuid_count(leaf, subleaf, out[0], out[1], out[2], out[3]); +} + +u64 xgetbv(unsigned int index) { + u32 eax = 0; + u32 edx = 0; + __asm__ volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(index)); + return static_cast(eax) | (static_cast(edx) << 32U); +} + +#endif + +bool bit_set(int value, int bit) { + return (value & (1 << bit)) != 0; +} +} // namespace + +CpuFeatures detect_cpu_features() { + CpuFeatures features{}; + + int regs[4]{}; + + cpuid(regs, 1, 0); + + const bool osxsave = bit_set(regs[2], 27); + const bool avx = bit_set(regs[2], 28); + + if (!osxsave || !avx) { + return features; + } + + const u64 xcr0 = xgetbv(0); + + const bool xmm_enabled = (xcr0 & 0x2) != 0; + const bool ymm_enabled = (xcr0 & 0x4) != 0; + const bool zmm_enabled = (xcr0 & 0x20) != 0 && (xcr0 & 0x40) != 0 && (xcr0 & 0x80) != 0; + + if (!xmm_enabled || !ymm_enabled) { + return features; + } + + cpuid(regs, 7, 0); + + features.avx2 = bit_set(regs[1], 5); + features.bmi2 = bit_set(regs[1], 8); + + if (zmm_enabled) { + features.avx512f = bit_set(regs[1], 16); + features.avx512dq = bit_set(regs[1], 29); + features.avx512bw = bit_set(regs[1], 30); + features.avx512vl = bit_set(regs[1], 31); + features.avx512vbmi = bit_set(regs[2], 1); + } + + return features; +} + +CpuFeatures get_cached_cpu_features() { + static const CpuFeatures features = detect_cpu_features(); + return features; +} +} // namespace pernix::internal +#else + +namespace pernix::internal { +CpuFeatures detect_cpu_features() { + return {}; +} + +CpuFeatures get_cached_cpu_features() { + static const CpuFeatures features = detect_cpu_features(); + return features; +} +} // namespace pernix::internal + +#endif diff --git a/src/dispatch/select.cpp b/src/dispatch/select.cpp new file mode 100644 index 0000000..e2851db --- /dev/null +++ b/src/dispatch/select.cpp @@ -0,0 +1,868 @@ +#include +#include + +namespace pernix::internal { +bool is_optional_fallback_backend(const Backend backend) { + switch (backend) { + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + default: + return false; + } +} + +bool is_compiled_backend(const Backend backend) { + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + return true; +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return true; +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return true; +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return true; +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return true; +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return true; +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return true; +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: + return true; +#endif + default: + return false; + } +} + +bool is_backend_supported_on_machine(const Backend backend) { + const CpuFeatures features = get_cached_cpu_features(); + + switch (backend) { + case Backend::Auto: + case Backend::Fallback: + case Backend::FallbackStdpar: + case Backend::FallbackSimd: + return true; + case Backend::X86Avx2: + return features.avx2; + case Backend::X86Bmi2: + return features.avx2 && features.bmi2; + case Backend::X86Avx512Vbmi: + return features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi; + case Backend::Arm64Neon: + return features.neon; + case Backend::Arm64Sve: + return features.sve2; + default: + return false; + } +} + +Kernel select_compress_block_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_block_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f32(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_compress_blocks_f32(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f32(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f32(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_blocks_f32(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f32(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_compress_block_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_block_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_block_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_block_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_block_f64(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_compress_blocks_f64(Backend backend, u8 bit_width, u32 block_size) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_compress_blocks_f64(bit_width, block_size); + case Backend::Fallback: + return select_fallback_compress_blocks_f64(bit_width, block_size); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_compress_blocks_f64(bit_width, block_size); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_compress_blocks_f64(bit_width, block_size); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_decompress_block_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_block_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f32(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_decompress_blocks_f32(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f32(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f32(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_decompress_block_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_block_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_block_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_block_f64(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_decompress_blocks_f64(Backend backend, u8 bit_width, u32 block_size, bool sign_values) { + if (is_optional_fallback_backend(backend) && !is_compiled_backend(backend)) { + return {"unsupported_implementation", nullptr}; + } + if (backend != Backend::Auto && backend != Backend::Fallback && is_compiled_backend(backend) && + !is_backend_supported_on_machine(backend)) { + return {"unsupported_implementation", nullptr}; + } + switch (backend) { + case Backend::Auto: + return select_auto_decompress_blocks_f64(bit_width, block_size, sign_values); + case Backend::Fallback: + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + case Backend::FallbackStdpar: + return select_fallback_stdpar_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_FALLBACK_SIMD) + case Backend::FallbackSimd: + return select_fallback_simd_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + case Backend::X86Avx512Vbmi: + return select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_AVX2) + case Backend::X86Avx2: + return select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_X86_BMI2) + case Backend::X86Bmi2: + return select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_NEON) + case Backend::Arm64Neon: + return select_neon_decompress_blocks_f64(bit_width, block_size, sign_values); +#endif +#if defined(PERNIX_BUILD_ARM64_SVE2) + case Backend::Arm64Sve: { + if (get_cached_cpu_features().sve2) { + return select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values); + } + return {"invalid_backend", nullptr}; + } +#endif + default: + return {"invalid_backend", nullptr}; + } +} + +Kernel select_auto_compress_block_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_block_f32(bit_width, block_size); +} + +Kernel select_auto_compress_blocks_f32(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f32(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_blocks_f32(bit_width, block_size); +} + +Kernel select_auto_compress_block_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_block_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_block_f64(bit_width, block_size); +} + +Kernel select_auto_compress_blocks_f64(u8 bit_width, u32 block_size) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_compress_blocks_f64(bit_width, block_size)) { + return kernel; + } + } +#endif + + return select_fallback_compress_blocks_f64(bit_width, block_size); +} + +Kernel select_auto_decompress_block_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_block_f32(bit_width, block_size, sign_values); +} + +Kernel select_auto_decompress_blocks_f32(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f32(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_blocks_f32(bit_width, block_size, sign_values); +} + +Kernel select_auto_decompress_block_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_block_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_block_f64(bit_width, block_size, sign_values); +} + +Kernel select_auto_decompress_blocks_f64(u8 bit_width, u32 block_size, bool sign_values) { +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) || defined(PERNIX_BUILD_X86_AVX2) || defined(PERNIX_BUILD_X86_BMI2) || \ + defined(PERNIX_BUILD_ARM64_NEON) || defined(PERNIX_BUILD_ARM64_SVE2) + const CpuFeatures features = get_cached_cpu_features(); +#endif + +#if defined(PERNIX_BUILD_X86_AVX512_VBMI) + if (features.avx512f && features.avx512dq && features.avx512bw && features.avx512vl && features.avx512vbmi) { + if (auto kernel = select_avx512vbmi_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_AVX2) + if (features.avx2) { + if (auto kernel = select_avx2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_X86_BMI2) + if (features.bmi2) { + if (auto kernel = select_bmi2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_NEON) + if (features.neon) { + if (auto kernel = select_neon_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + +#if defined(PERNIX_BUILD_ARM64_SVE2) + if (features.sve2) { + if (auto kernel = select_sve2_decompress_blocks_f64(bit_width, block_size, sign_values)) { + return kernel; + } + } +#endif + + return select_fallback_decompress_blocks_f64(bit_width, block_size, sign_values); +} +} // namespace pernix::internal diff --git a/src/fallback/compression.cpp b/src/fallback/compression.cpp deleted file mode 100644 index 1ed56e8..0000000 --- a/src/fallback/compression.cpp +++ /dev/null @@ -1,149 +0,0 @@ -#include - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_COMPRESS_BLOCK_CASE(N) \ - case N: \ - return compress_block_fallback(input, scale, output); - -#define PERNIX_COMPRESS_BLOCKS_CASE(N) \ - case N: \ - return compress_blocks_fallback(input, scale, output, blocks); - -int compress_block_fallback(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int compress_block_fallback_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int compress_blocks_fallback(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int compress_blocks_fallback_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus diff --git a/src/fallback/decompression.cpp b/src/fallback/decompression.cpp deleted file mode 100644 index e43b1df..0000000 --- a/src/fallback/decompression.cpp +++ /dev/null @@ -1,150 +0,0 @@ -#include - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return decompress_block_fallback(input, scale, output); - -#define PERNIX_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return decompress_blocks_fallback(input, scale, output, blocks); - -int decompress_block_fallback(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int decompress_block_fallback_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int decompress_blocks_fallback(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int decompress_blocks_fallback_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_DECOMPRESS_BLOCK_CASE -#undef PERNIX_DECOMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus \ No newline at end of file diff --git a/src/fallback/fallback_compression.cpp b/src/fallback/fallback_compression.cpp new file mode 100644 index 0000000..1e3a075 --- /dev/null +++ b/src/fallback/fallback_compression.cpp @@ -0,0 +1,202 @@ +#include +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("fallback", &compress_block_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("fallback", &compress_blocks_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("fallback", &compress_block_fallback) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("fallback", &compress_blocks_fallback) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } + +Kernel select_fallback_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +Kernel select_fallback_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +Kernel select_fallback_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +Kernel select_fallback_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal diff --git a/src/fallback/fallback_decompression.cpp b/src/fallback/fallback_decompression.cpp new file mode 100644 index 0000000..dfedb75 --- /dev/null +++ b/src/fallback/fallback_decompression.cpp @@ -0,0 +1,213 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_block_fallback); \ + return Kernel("fallback", &decompress_block_fallback) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback", &decompress_blocks_fallback); \ + return Kernel("fallback", &decompress_blocks_fallback) + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback", nullptr}; \ + } \ + break + +Kernel select_fallback_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"fallback", nullptr}; + } +} + +Kernel select_fallback_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"fallback", nullptr}; + } +} + +Kernel select_fallback_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"fallback", nullptr}; + } +} + +Kernel select_fallback_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"fallback", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal diff --git a/src/fallback/simd_compression.cpp b/src/fallback/simd_compression.cpp new file mode 100644 index 0000000..cb9a467 --- /dev/null +++ b/src/fallback/simd_compression.cpp @@ -0,0 +1,4 @@ +#include + +// Header-only selector definitions live in select_impl.h. +// This translation unit reserves the compiled std::simd compression backend slot. diff --git a/src/fallback/simd_decompression.cpp b/src/fallback/simd_decompression.cpp new file mode 100644 index 0000000..fe09cc3 --- /dev/null +++ b/src/fallback/simd_decompression.cpp @@ -0,0 +1,4 @@ +#include + +// Header-only selector definitions live in select_impl.h. +// This translation unit reserves the compiled std::simd decompression backend slot. diff --git a/src/fallback/stdpar_compression.cpp b/src/fallback/stdpar_compression.cpp new file mode 100644 index 0000000..7d982de --- /dev/null +++ b/src/fallback/stdpar_compression.cpp @@ -0,0 +1,201 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("fallback_stdpar", &compress_blocks_fallback_stdpar) + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +Kernel select_fallback_stdpar_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_STDPAR_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal diff --git a/src/fallback/stdpar_decompression.cpp b/src/fallback/stdpar_decompression.cpp new file mode 100644 index 0000000..ec0dd7b --- /dev/null +++ b/src/fallback/stdpar_decompression.cpp @@ -0,0 +1,209 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_block_fallback_stdpar) + +#define PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar); \ + return Kernel("fallback_stdpar", &decompress_blocks_fallback_stdpar) + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"fallback_stdpar", nullptr}; \ + } + +Kernel select_fallback_stdpar_decompress_block_f32(const u8 bit_width, const u32 block_size, const bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_decompress_blocks_f32(const u8 bit_width, const u32 block_size, const bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_decompress_block_f64(const u8 bit_width, const u32 block_size, const bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +Kernel select_fallback_stdpar_decompress_blocks_f64(const u8 bit_width, const u32 block_size, const bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"fallback_stdpar", nullptr}; + } +} + +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_STDPAR_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_STDPAR_DECOMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal diff --git a/src/pernix.cpp b/src/pernix.cpp index 87ccf9d..0479c25 100644 --- a/src/pernix.cpp +++ b/src/pernix.cpp @@ -1,124 +1,107 @@ -#include +#include -#ifdef __cplusplus -namespace pernix { extern "C" { -#endif - -// Use the best available implementation based on detected CPU features at compile time -#ifdef PERNIX_AVX2_ENABLED -#ifdef PERNIX_AVX512_VBMI_ENABLED -int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return mm512_compress_block_avx512vbmi(bit_width, input, scale, output); +u8 pernix_min_bit_width(void) { + return pernix::detail::min_bit_width; } -int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return mm512_compress_block_f64_avx512vbmi(bit_width, input, scale, output); +u8 pernix_max_bit_width(void) { + return pernix::detail::max_bit_width; } -int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return mm512_compress_blocks_avx512vbmi(bit_width, input, scale, output, blocks); +bool pernix_is_valid_bit_width(u8 bit_width) { + return pernix::detail::is_valid_bit_width(bit_width); } -int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return mm512_compress_blocks_f64_avx512vbmi(bit_width, input, scale, output, blocks); +bool pernix_is_valid_block_size(u32 block_size) { + return pernix::detail::is_valid_block_size(block_size); } -int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return mm512_decompress_block_avx512vbmi(bit_width, input, scale, output); +u32 pernix_compressed_block_size(void) { + return pernix::detail::fixed_block_size; } -int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return mm512_decompress_block_f64_avx512vbmi(bit_width, input, scale, output); +u32 pernix_elements_per_block(u8 bit_width) { + return pernix::detail::elements_per_block(bit_width); } -int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - return mm512_decompress_blocks_avx512vbmi(bit_width, input, scale, output, blocks); +const char* pernix_status_string(pernix_status status) { + return pernix::detail::status_string(status); } -int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - return mm512_decompress_blocks_f64_avx512vbmi(bit_width, input, scale, output, blocks); -} -#else -int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return mm256_compress_block_avx2(bit_width, input, scale, output); +pernix_status pernix_scale_f32(float bmax, u8 bit_width, float* scale) { + return pernix::detail::scale_from_bmax(bmax, bit_width, scale); } -int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return mm256_compress_block_f64_avx2(bit_width, input, scale, output); +pernix_status pernix_scale_f64(double bmax, u8 bit_width, double* scale) { + return pernix::detail::scale_from_bmax(bmax, bit_width, scale); } -int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return mm256_compress_blocks_avx2(bit_width, input, scale, output, blocks); +pernix_status pernix_decompression_scale_f32(float bmax, u8 bit_width, float* scale) { + return pernix_scale_f32(bmax, bit_width, scale); } -int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return mm256_compress_blocks_f64_avx2(bit_width, input, scale, output, blocks); +pernix_status pernix_decompression_scale_f64(double bmax, u8 bit_width, double* scale) { + return pernix_scale_f64(bmax, bit_width, scale); } -int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return mm256_decompress_block_avx2(bit_width, input, scale, output); +pernix_status pernix_inverse_scale_f32(float scale, float* inverse_scale) { + return pernix::detail::inverse_scale(scale, inverse_scale); } -int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return mm256_decompress_block_f64_avx2(bit_width, input, scale, output); +pernix_status pernix_inverse_scale_f64(double scale, double* inverse_scale) { + return pernix::detail::inverse_scale(scale, inverse_scale); } -int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - return mm256_decompress_blocks_avx2(bit_width, input, scale, output, blocks); +pernix_status pernix_compression_scale_f32(float bmax, u8 bit_width, float* inverse_scale) { + return pernix::detail::compression_scale_from_bmax(bmax, bit_width, inverse_scale); } -int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - return mm256_decompress_blocks_f64_avx2(bit_width, input, scale, output, blocks); -} -#endif -#else -int compress_block(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output) { - return compress_block_fallback(bit_width, input, scale, output); +pernix_status pernix_compression_scale_f64(double bmax, u8 bit_width, double* inverse_scale) { + return pernix::detail::compression_scale_from_bmax(bmax, bit_width, inverse_scale); } -int compress_block_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output) { - return compress_block_fallback_f64(bit_width, input, scale, output); +pernix_status pernix_compress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output) { + return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, output); } -int compress_blocks(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return compress_blocks_fallback(bit_width, input, scale, output, blocks); +pernix_status pernix_compress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, void* output, + u32 blocks) { + return pernix::detail::compress_blocks(static_cast(backend), bit_width, block_size, input, scale, output, blocks); } -int compress_blocks_f64(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, uint8_t* __restrict__ output, - const uint32_t blocks) { - return compress_blocks_fallback_f64(bit_width, input, scale, output, blocks); +pernix_status pernix_decompress_block_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output, bool sign_values) { + return pernix::detail::decompress_block(static_cast(backend), bit_width, block_size, input, scale, output, + sign_values); } -int decompress_block(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output) { - return decompress_block_fallback(bit_width, input, scale, output); +pernix_status pernix_decompress_blocks_f32(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, float scale, + void* output, u32 blocks, bool sign_values) { + return pernix::detail::decompress_blocks(static_cast(backend), bit_width, block_size, input, scale, output, blocks, + sign_values); } -int decompress_block_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output) { - return decompress_block_fallback_f64(bit_width, input, scale, output); +pernix_status pernix_compress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output) { + return pernix::detail::compress_block(static_cast(backend), bit_width, block_size, input, scale, output); } -int decompress_blocks(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - return decompress_blocks_fallback(bit_width, input, scale, output, blocks); +pernix_status pernix_compress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, u32 blocks) { + return pernix::detail::compress_blocks(static_cast(backend), bit_width, block_size, input, scale, output, blocks); } -int decompress_blocks_f64(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, double_t* __restrict__ output, - const uint32_t blocks) { - return decompress_blocks_fallback_f64(bit_width, input, scale, output, blocks); +pernix_status pernix_decompress_block_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, bool sign_values) { + return pernix::detail::decompress_block(static_cast(backend), bit_width, block_size, input, scale, output, + sign_values); } -#endif -#ifdef __cplusplus +pernix_status pernix_decompress_blocks_f64(pernix_backend backend, u8 bit_width, u32 block_size, const void* input, double scale, + void* output, u32 blocks, bool sign_values) { + return pernix::detail::decompress_blocks(static_cast(backend), bit_width, block_size, input, scale, output, blocks, + sign_values); +} } -} // namespace pernix -#endif // __cplusplus \ No newline at end of file diff --git a/src/x86/avx2/avx2_compression.cpp b/src/x86/avx2/avx2_compression.cpp new file mode 100644 index 0000000..0ce0189 --- /dev/null +++ b/src/x86/avx2/avx2_compression.cpp @@ -0,0 +1,201 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("avx2", &mm256_compress_block_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("avx2", &mm256_compress_blocks_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("avx2", &mm256_compress_block_avx2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("avx2", &mm256_compress_blocks_avx2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } + +Kernel select_avx2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +Kernel select_avx2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +Kernel select_avx2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } +} + +Kernel select_avx2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal diff --git a/src/x86/avx2/avx2_decompression.cpp b/src/x86/avx2/avx2_decompression.cpp new file mode 100644 index 0000000..4629ba1 --- /dev/null +++ b/src/x86/avx2/avx2_decompression.cpp @@ -0,0 +1,212 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_block_avx2); \ + return Kernel("avx2", &mm256_decompress_block_avx2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx2", &mm256_decompress_blocks_avx2); \ + return Kernel("avx2", &mm256_decompress_blocks_avx2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx2", nullptr}; \ + } \ + break + +Kernel select_avx2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"avx2", nullptr}; + } +} + +Kernel select_avx2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"avx2", nullptr}; + } +} + +Kernel select_avx2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"avx2", nullptr}; + } +} + +Kernel select_avx2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"avx2", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal diff --git a/src/x86/avx2/compression.cpp b/src/x86/avx2/compression.cpp deleted file mode 100644 index 231fc75..0000000 --- a/src/x86/avx2/compression.cpp +++ /dev/null @@ -1,154 +0,0 @@ -#include -#include - -#ifdef PERNIX_AVX2_ENABLED - -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_COMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm256_compress_block_avx2(input, scale, output); - -#define PERNIX_COMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm256_compress_blocks_avx2(input, scale, output, blocks); - -int mm256_compress_block_avx2(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_compress_block_f64_avx2(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_compress_blocks_avx2(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm256_compress_blocks_f64_avx2(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED diff --git a/src/x86/avx2/decompression.cpp b/src/x86/avx2/decompression.cpp deleted file mode 100644 index 47c473a..0000000 --- a/src/x86/avx2/decompression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#ifdef PERNIX_AVX2_ENABLED -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm256_decompress_block_avx2(input, scale, output); - -#define PERNIX_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm256_decompress_blocks_avx2(input, scale, output, blocks); - -int mm256_decompress_block_avx2(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_block_f64_avx2(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_blocks_avx2(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_blocks_f64_avx2(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED \ No newline at end of file diff --git a/src/x86/avx512vbmi/avx512vbmi_compression.cpp b/src/x86/avx512vbmi/avx512vbmi_compression.cpp new file mode 100644 index 0000000..03e613f --- /dev/null +++ b/src/x86/avx512vbmi/avx512vbmi_compression.cpp @@ -0,0 +1,201 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("avx512vbmi", &mm512_compress_block_avx512vbmi) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("avx512vbmi", &mm512_compress_blocks_avx512vbmi) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } + +Kernel select_avx512vbmi_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal diff --git a/src/x86/avx512vbmi/avx512vbmi_decompression.cpp b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp new file mode 100644 index 0000000..a3dbc13 --- /dev/null +++ b/src/x86/avx512vbmi/avx512vbmi_decompression.cpp @@ -0,0 +1,212 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_block_avx512vbmi) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi); \ + return Kernel("avx512vbmi", &mm512_decompress_blocks_avx512vbmi) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"avx512vbmi", nullptr}; \ + } \ + break + +Kernel select_avx512vbmi_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +Kernel select_avx512vbmi_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"avx512vbmi", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal diff --git a/src/x86/avx512vbmi/compression.cpp b/src/x86/avx512vbmi/compression.cpp deleted file mode 100644 index bdc6635..0000000 --- a/src/x86/avx512vbmi/compression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#if defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_AVX512_VBMI_ENABLED) -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_COMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm512_compress_block_avx512vbmi(input, scale, output); - -#define PERNIX_COMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm512_compress_blocks_avx512vbmi(input, scale, output, blocks); - -int mm512_compress_block_avx512vbmi(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm512_compress_block_f64_avx512vbmi(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm512_compress_blocks_avx512vbmi(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm512_compress_blocks_f64_avx512vbmi(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_AVX512_VBMI_ENABLED) \ No newline at end of file diff --git a/src/x86/avx512vbmi/decompression.cpp b/src/x86/avx512vbmi/decompression.cpp deleted file mode 100644 index 1273f6a..0000000 --- a/src/x86/avx512vbmi/decompression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#if defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_AVX512_VBMI_ENABLED) -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm512_decompress_block_avx512vbmi(input, scale, output); - -#define PERNIX_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm512_decompress_blocks_avx512vbmi(input, scale, output, blocks); - -int mm512_decompress_block_avx512vbmi(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm512_decompress_block_f64_avx512vbmi(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm512_decompress_blocks_avx512vbmi(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm512_decompress_blocks_f64_avx512vbmi(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED && PERNIX_AVX512_VBMI_ENABLED \ No newline at end of file diff --git a/src/x86/bmi2/bmi2_compression.cpp b/src/x86/bmi2/bmi2_compression.cpp new file mode 100644 index 0000000..9920ced --- /dev/null +++ b/src/x86/bmi2/bmi2_compression.cpp @@ -0,0 +1,201 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_COMPRESS_BLOCK_32(N, BS) \ + case N: \ + return Kernel("bmi2", &mm256_compress_block_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_32(N, BS) \ + case N: \ + return Kernel("bmi2", &mm256_compress_blocks_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCK_64(N, BS) \ + case N: \ + return Kernel("bmi2", &mm256_compress_block_bmi2) + +#define PERNIX_CASE_COMPRESS_BLOCKS_64(N, BS) \ + case N: \ + return Kernel("bmi2", &mm256_compress_blocks_bmi2) + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } + +#define PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_COMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_COMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } + +Kernel select_bmi2_compress_block_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_compress_blocks_f32(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_compress_block_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_compress_blocks_f64(const u8 bit_width, const u32 block_size) { + switch (block_size) { + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +#undef PERNIX_CASE_COMPRESS_BLOCK_32 +#undef PERNIX_CASE_COMPRESS_BLOCKS_32 +#undef PERNIX_CASE_COMPRESS_BLOCK_64 +#undef PERNIX_CASE_COMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_COMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_COMPRESS_BLOCKS_SWITCH_64 +} // namespace pernix::internal diff --git a/src/x86/bmi2/bmi2_decompression.cpp b/src/x86/bmi2/bmi2_decompression.cpp new file mode 100644 index 0000000..d02fc3b --- /dev/null +++ b/src/x86/bmi2/bmi2_decompression.cpp @@ -0,0 +1,212 @@ +#include +#include + +namespace pernix::internal { +#define PERNIX_CASE_DECOMPRESS_BLOCK_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_32(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCK_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_block_bmi2); \ + return Kernel("bmi2", &mm256_decompress_block_bmi2) + +#define PERNIX_CASE_DECOMPRESS_BLOCKS_64(N, BS) \ + case N: \ + if (sign_values) \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2); \ + return Kernel("bmi2", &mm256_decompress_blocks_bmi2) +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_32(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCK_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ + break + +#define PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(BS) \ + case BS: \ + switch (bit_width) { \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(1, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(2, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(3, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(4, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(5, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(6, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(7, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(8, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(9, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(10, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(11, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(12, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(13, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(14, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(15, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(16, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(17, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(18, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(19, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(20, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(21, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(22, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(23, BS); \ + PERNIX_CASE_DECOMPRESS_BLOCKS_64(24, BS); \ + default: \ + return {"bmi2", nullptr}; \ + } \ + break + +Kernel select_bmi2_decompress_block_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_decompress_blocks_f32(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32(1024); + default: + return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_decompress_block_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +Kernel select_bmi2_decompress_blocks_f64(const u8 bit_width, const u32 block_size, bool sign_values) { + switch (block_size) { + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(64); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(128); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(256); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(512); + PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64(1024); + default: + return {"bmi2", nullptr}; + } +} + +#undef PERNIX_CASE_DECOMPRESS_BLOCK_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_32 +#undef PERNIX_CASE_DECOMPRESS_BLOCK_64 +#undef PERNIX_CASE_DECOMPRESS_BLOCKS_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_32 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_64 +#undef PERNIX_BLOCK_SIZE_DECOMPRESS_SWITCH_BLOCKS_64 +} // namespace pernix::internal diff --git a/src/x86/bmi2/compression.cpp b/src/x86/bmi2/compression.cpp deleted file mode 100644 index 79c43d8..0000000 --- a/src/x86/bmi2/compression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#if defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_BMI2_ENABLED) -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_COMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm256_compress_block_bmi2(input, scale, output); - -#define PERNIX_COMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm256_compress_blocks_bmi2(input, scale, output, blocks); - -int mm256_compress_block_bmi2(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_compress_block_f64_bmi2(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCK_CASE(1) - PERNIX_COMPRESS_BLOCK_CASE(2) - PERNIX_COMPRESS_BLOCK_CASE(3) - PERNIX_COMPRESS_BLOCK_CASE(4) - PERNIX_COMPRESS_BLOCK_CASE(5) - PERNIX_COMPRESS_BLOCK_CASE(6) - PERNIX_COMPRESS_BLOCK_CASE(7) - PERNIX_COMPRESS_BLOCK_CASE(8) - PERNIX_COMPRESS_BLOCK_CASE(9) - PERNIX_COMPRESS_BLOCK_CASE(10) - PERNIX_COMPRESS_BLOCK_CASE(11) - PERNIX_COMPRESS_BLOCK_CASE(12) - PERNIX_COMPRESS_BLOCK_CASE(13) - PERNIX_COMPRESS_BLOCK_CASE(14) - PERNIX_COMPRESS_BLOCK_CASE(15) - PERNIX_COMPRESS_BLOCK_CASE(16) - PERNIX_COMPRESS_BLOCK_CASE(17) - PERNIX_COMPRESS_BLOCK_CASE(18) - PERNIX_COMPRESS_BLOCK_CASE(19) - PERNIX_COMPRESS_BLOCK_CASE(20) - PERNIX_COMPRESS_BLOCK_CASE(21) - PERNIX_COMPRESS_BLOCK_CASE(22) - PERNIX_COMPRESS_BLOCK_CASE(23) - PERNIX_COMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_compress_blocks_bmi2(const uint8_t bit_width, const float_t* __restrict__ input, const float_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm256_compress_blocks_f64_bmi2(const uint8_t bit_width, const double_t* __restrict__ input, const double_t scale, - uint8_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_COMPRESS_BLOCKS_CASE(1) - PERNIX_COMPRESS_BLOCKS_CASE(2) - PERNIX_COMPRESS_BLOCKS_CASE(3) - PERNIX_COMPRESS_BLOCKS_CASE(4) - PERNIX_COMPRESS_BLOCKS_CASE(5) - PERNIX_COMPRESS_BLOCKS_CASE(6) - PERNIX_COMPRESS_BLOCKS_CASE(7) - PERNIX_COMPRESS_BLOCKS_CASE(8) - PERNIX_COMPRESS_BLOCKS_CASE(9) - PERNIX_COMPRESS_BLOCKS_CASE(10) - PERNIX_COMPRESS_BLOCKS_CASE(11) - PERNIX_COMPRESS_BLOCKS_CASE(12) - PERNIX_COMPRESS_BLOCKS_CASE(13) - PERNIX_COMPRESS_BLOCKS_CASE(14) - PERNIX_COMPRESS_BLOCKS_CASE(15) - PERNIX_COMPRESS_BLOCKS_CASE(16) - PERNIX_COMPRESS_BLOCKS_CASE(17) - PERNIX_COMPRESS_BLOCKS_CASE(18) - PERNIX_COMPRESS_BLOCKS_CASE(19) - PERNIX_COMPRESS_BLOCKS_CASE(20) - PERNIX_COMPRESS_BLOCKS_CASE(21) - PERNIX_COMPRESS_BLOCKS_CASE(22) - PERNIX_COMPRESS_BLOCKS_CASE(23) - PERNIX_COMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED && PERNIX_BMI2_ENABLED diff --git a/src/x86/bmi2/decompression.cpp b/src/x86/bmi2/decompression.cpp deleted file mode 100644 index c9cbad7..0000000 --- a/src/x86/bmi2/decompression.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include - -#if defined(PERNIX_AVX2_ENABLED) && defined(PERNIX_BMI2_ENABLED) -#ifdef __cplusplus -namespace pernix { -extern "C" { -#endif - -#define PERNIX_DECOMPRESS_BLOCK_CASE(N) \ - case N: \ - return mm256_decompress_block_bmi2(input, scale, output); - -#define PERNIX_DECOMPRESS_BLOCKS_CASE(N) \ - case N: \ - return mm256_decompress_blocks_bmi2(input, scale, output, blocks); - -int mm256_decompress_block_bmi2(const uint8_t bit_width, const uint8_t* __restrict__ input, const float_t scale, - float_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_block_f64_bmi2(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCK_CASE(1) - PERNIX_DECOMPRESS_BLOCK_CASE(2) - PERNIX_DECOMPRESS_BLOCK_CASE(3) - PERNIX_DECOMPRESS_BLOCK_CASE(4) - PERNIX_DECOMPRESS_BLOCK_CASE(5) - PERNIX_DECOMPRESS_BLOCK_CASE(6) - PERNIX_DECOMPRESS_BLOCK_CASE(7) - PERNIX_DECOMPRESS_BLOCK_CASE(8) - PERNIX_DECOMPRESS_BLOCK_CASE(9) - PERNIX_DECOMPRESS_BLOCK_CASE(10) - PERNIX_DECOMPRESS_BLOCK_CASE(11) - PERNIX_DECOMPRESS_BLOCK_CASE(12) - PERNIX_DECOMPRESS_BLOCK_CASE(13) - PERNIX_DECOMPRESS_BLOCK_CASE(14) - PERNIX_DECOMPRESS_BLOCK_CASE(15) - PERNIX_DECOMPRESS_BLOCK_CASE(16) - PERNIX_DECOMPRESS_BLOCK_CASE(17) - PERNIX_DECOMPRESS_BLOCK_CASE(18) - PERNIX_DECOMPRESS_BLOCK_CASE(19) - PERNIX_DECOMPRESS_BLOCK_CASE(20) - PERNIX_DECOMPRESS_BLOCK_CASE(21) - PERNIX_DECOMPRESS_BLOCK_CASE(22) - PERNIX_DECOMPRESS_BLOCK_CASE(23) - PERNIX_DECOMPRESS_BLOCK_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_blocks_bmi2(const uint8_t bit_width, const uint8_t* __restrict__ input, float_t scale, float_t* __restrict__ output, - const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -int mm256_decompress_blocks_f64_bmi2(const uint8_t bit_width, const uint8_t* __restrict__ input, const double_t scale, - double_t* __restrict__ output, const uint32_t blocks) { - switch (bit_width) { - PERNIX_DECOMPRESS_BLOCKS_CASE(1) - PERNIX_DECOMPRESS_BLOCKS_CASE(2) - PERNIX_DECOMPRESS_BLOCKS_CASE(3) - PERNIX_DECOMPRESS_BLOCKS_CASE(4) - PERNIX_DECOMPRESS_BLOCKS_CASE(5) - PERNIX_DECOMPRESS_BLOCKS_CASE(6) - PERNIX_DECOMPRESS_BLOCKS_CASE(7) - PERNIX_DECOMPRESS_BLOCKS_CASE(8) - PERNIX_DECOMPRESS_BLOCKS_CASE(9) - PERNIX_DECOMPRESS_BLOCKS_CASE(10) - PERNIX_DECOMPRESS_BLOCKS_CASE(11) - PERNIX_DECOMPRESS_BLOCKS_CASE(12) - PERNIX_DECOMPRESS_BLOCKS_CASE(13) - PERNIX_DECOMPRESS_BLOCKS_CASE(14) - PERNIX_DECOMPRESS_BLOCKS_CASE(15) - PERNIX_DECOMPRESS_BLOCKS_CASE(16) - PERNIX_DECOMPRESS_BLOCKS_CASE(17) - PERNIX_DECOMPRESS_BLOCKS_CASE(18) - PERNIX_DECOMPRESS_BLOCKS_CASE(19) - PERNIX_DECOMPRESS_BLOCKS_CASE(20) - PERNIX_DECOMPRESS_BLOCKS_CASE(21) - PERNIX_DECOMPRESS_BLOCKS_CASE(22) - PERNIX_DECOMPRESS_BLOCKS_CASE(23) - PERNIX_DECOMPRESS_BLOCKS_CASE(24) - default: - return -1; - } -} - -#undef PERNIX_COMPRESS_BLOCK_CASE -#undef PERNIX_COMPRESS_BLOCKS_CASE - -#ifdef __cplusplus -} -} // namespace pernix -#endif // __cplusplus -#endif // PERNIX_AVX2_ENABLED && PERNIX_BMI2_ENABLED \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 370ec3b..c18c3ef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,18 +1,8 @@ find_package(PkgConfig) pkg_search_module(GTEST REQUIRED gtest) -include(CheckCXXCompilerFlag) -file(GLOB_RECURSE - SOURCE_FILES - CONFIGURE_DEPENDS - *.cpp -) - -file(GLOB_RECURSE - HEADER_FILES - CONFIGURE_DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/include/*.h -) +file(GLOB PERNIX_TEST_SOURCE_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) +list(FILTER PERNIX_TEST_SOURCE_FILES EXCLUDE REGEX ".*/header_only_tests\\.cpp$") set(PERNIX_TEST_BLOCK_SIZES 64 128 256 512 CACHE STRING "Block sizes to build as separate test executables") set(PERNIX_TEST_TARGETS) @@ -22,20 +12,48 @@ foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) list(APPEND PERNIX_TEST_TARGETS ${TEST_TARGET}) add_executable(${TEST_TARGET}) - target_sources(${TEST_TARGET} PRIVATE ${SOURCE_FILES} ${HEADER_FILES}) + target_sources(${TEST_TARGET} PRIVATE ${PERNIX_TEST_SOURCE_FILES}) target_link_libraries(${TEST_TARGET} PRIVATE pernix ${GTEST_LDFLAGS}) target_compile_features(${TEST_TARGET} PRIVATE cxx_std_20) target_compile_options(${TEST_TARGET} PRIVATE ${GTEST_CFLAGS}) target_compile_definitions(${TEST_TARGET} PRIVATE PERNIX_TEST_BLOCK_SIZE=${BLOCK_SIZE}) target_include_directories(${TEST_TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) + + set(HEADER_ONLY_TARGET pernix_header_only_tests_${BLOCK_SIZE}) + list(APPEND PERNIX_TEST_TARGETS ${HEADER_ONLY_TARGET}) + add_executable(${HEADER_ONLY_TARGET} header_only_tests.cpp main.cpp) + target_link_libraries(${HEADER_ONLY_TARGET} PRIVATE pernix_header_only ${GTEST_LDFLAGS}) + target_compile_features(${HEADER_ONLY_TARGET} PRIVATE cxx_std_20) + target_compile_options(${HEADER_ONLY_TARGET} PRIVATE ${GTEST_CFLAGS}) + target_compile_definitions(${HEADER_ONLY_TARGET} PRIVATE PERNIX_TEST_BLOCK_SIZE=${BLOCK_SIZE}) + target_include_directories(${HEADER_ONLY_TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) endforeach () add_custom_target(pernix_tests DEPENDS ${PERNIX_TEST_TARGETS}) -# check_cxx_compiler_flag(-O3 HAS_OPTIMIZE3_FLAG) -# if (HAS_OPTIMIZE3_FLAG) -# target_compile_options(pernix_tests PRIVATE -O3) -# endif () +add_executable(pernix_c_api_roundtrip c_api_roundtrip.c) +target_link_libraries(pernix_c_api_roundtrip PRIVATE pernix) +target_include_directories(pernix_c_api_roundtrip PRIVATE ${PROJECT_SOURCE_DIR}/include) +add_test(NAME pernix_c_api_roundtrip COMMAND pernix_c_api_roundtrip) + +add_executable(pernix_c_api_validation c_api_validation.c) +target_link_libraries(pernix_c_api_validation PRIVATE pernix) +target_include_directories(pernix_c_api_validation PRIVATE ${PROJECT_SOURCE_DIR}/include) +add_test(NAME pernix_c_api_validation COMMAND pernix_c_api_validation) + +if (PERNIX_ENABLE_INSTALL_CONSUMER_TESTS) + add_test(NAME pernix_install_consumer + COMMAND ${CMAKE_COMMAND} + -DPERNIX_BUILD_DIR=${PROJECT_BINARY_DIR} + -DPERNIX_CONSUMER_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/install_consumer + -DPERNIX_CONSUMER_BUILD_DIR=${PROJECT_BINARY_DIR}/install-consumer-build + -DPERNIX_INSTALL_PREFIX=${PROJECT_BINARY_DIR}/install-consumer-prefix + -DPERNIX_TEST_CONFIG=$ + -DPERNIX_C_COMPILER=${CMAKE_C_COMPILER} + -DPERNIX_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -P ${CMAKE_CURRENT_SOURCE_DIR}/install_consumer/run.cmake + ) +endif () include(GoogleTest) foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) @@ -45,18 +63,21 @@ foreach (BLOCK_SIZE IN LISTS PERNIX_TEST_BLOCK_SIZES) NO_PRETTY_TYPES PROPERTIES LABELS "unit;block_size_${BLOCK_SIZE}" ) + gtest_discover_tests(pernix_header_only_tests_${BLOCK_SIZE} + DISCOVERY_MODE PRE_TEST + EXTRA_ARGS --gtest_color=yes + PROPERTIES LABELS "unit;header_only;block_size_${BLOCK_SIZE}" + ) endforeach () if (PERNIX_ENABLE_CODE_COVERAGE) message(STATUS "Code coverage enabled for tests") - # set compile and link options for code coverage foreach (TEST_TARGET IN LISTS PERNIX_TEST_TARGETS) target_compile_options(${TEST_TARGET} PRIVATE -g -O0 --coverage) target_link_options(${TEST_TARGET} PRIVATE --coverage) endforeach () - # find lcov and genhtml find_program(LCOV_PROGRAM lcov) find_program(GENHTML_PROGRAM genhtml) if (NOT LCOV_PROGRAM OR NOT GENHTML_PROGRAM) diff --git a/tests/c_api_roundtrip.c b/tests/c_api_roundtrip.c new file mode 100644 index 0000000..0ffe099 --- /dev/null +++ b/tests/c_api_roundtrip.c @@ -0,0 +1,55 @@ +#include + +int main(void) { + enum { + bit_width = 16, + block_size = 64, + elements = (block_size * 8) / bit_width + }; + + if (pernix_min_bit_width() != 1 || pernix_max_bit_width() != 24) { + return 1; + } + if (!pernix_is_valid_bit_width(bit_width) || pernix_is_valid_bit_width(25)) { + return 1; + } + if (pernix_compressed_block_size() != 64 || pernix_elements_per_block(bit_width) != elements) { + return 1; + } + + float input[elements]; + float restored[elements]; + u8 compressed[block_size]; + float bmax = 0.0f; + + for (int i = 0; i < elements; ++i) { + input[i] = ((float)i - 16.0f) * 0.125f; + const float magnitude = input[i] < 0.0f ? -input[i] : input[i]; + bmax = bmax < magnitude ? magnitude : bmax; + } + + float scale = 0.0f; + if (pernix_scale_f32(bmax, bit_width, &scale) != PERNIX_STATUS_OK) { + return 1; + } + + pernix_status status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 1.0f / scale, compressed); + if (status != PERNIX_STATUS_OK) { + return 1; + } + + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, scale, restored, true); + if (status != PERNIX_STATUS_OK) { + return 2; + } + + for (int i = 0; i < elements; ++i) { + const float diff = restored[i] - input[i]; + const float error = diff < 0.0f ? -diff : diff; + if (error > scale) { + return 3; + } + } + + return 0; +} diff --git a/tests/c_api_validation.c b/tests/c_api_validation.c new file mode 100644 index 0000000..ad03608 --- /dev/null +++ b/tests/c_api_validation.c @@ -0,0 +1,147 @@ +#include + +#include +#include + +enum { + bit_width = 8, + block_size = 64, + elements = (block_size * 8) / bit_width +}; + +static int expect_status(pernix_status actual, pernix_status expected) { + return actual == expected ? 0 : 1; +} + +int main(void) { + float input_f32[elements] = {0}; + float output_f32[elements] = {0}; + double input_f64[elements] = {0}; + double output_f64[elements] = {0}; + u8 compressed[block_size] = {0}; + float scale_f32 = 0.0f; + double scale_f64 = 0.0; + + if (expect_status(pernix_scale_f32(1.0f, bit_width, &scale_f32), PERNIX_STATUS_OK) != 0) { + return 1; + } + if (expect_status(pernix_scale_f64(1.0, bit_width, &scale_f64), PERNIX_STATUS_OK) != 0) { + return 2; + } + if (expect_status(pernix_scale_f32(1.0f, bit_width, 0), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 3; + } + if (expect_status(pernix_scale_f32(-1.0f, bit_width, &scale_f32), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 4; + } + if (expect_status(pernix_scale_f64(INFINITY, bit_width, &scale_f64), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 5; + } + + if (!pernix_is_valid_bit_width(pernix_min_bit_width()) || !pernix_is_valid_bit_width(pernix_max_bit_width()) || + pernix_is_valid_bit_width(0) || pernix_is_valid_bit_width((u8)(pernix_max_bit_width() + 1U))) { + return 6; + } + if (pernix_compressed_block_size() != block_size || pernix_elements_per_block(bit_width) != elements) { + return 7; + } + if (!pernix_is_valid_block_size(block_size) || pernix_is_valid_block_size(96)) { + return 8; + } + + if (strcmp(pernix_status_string(PERNIX_STATUS_OK), "PERNIX_STATUS_OK") != 0 || + strcmp(pernix_status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION), "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION") != 0 || + strcmp(pernix_status_string((pernix_status)123), "PERNIX_STATUS_UNKNOWN") != 0) { + return 23; + } + + float decompression_scale_f32 = 0.0f; + float compression_scale_f32 = 0.0f; + float inverse_scale_f32 = 0.0f; + if (expect_status(pernix_decompression_scale_f32(127.0f, bit_width, &decompression_scale_f32), PERNIX_STATUS_OK) != 0 || + expect_status(pernix_compression_scale_f32(127.0f, bit_width, &compression_scale_f32), PERNIX_STATUS_OK) != 0 || + expect_status(pernix_inverse_scale_f32(decompression_scale_f32, &inverse_scale_f32), PERNIX_STATUS_OK) != 0) { + return 24; + } + if (fabsf(decompression_scale_f32 - 1.0f) > 0.0f || fabsf(compression_scale_f32 - 1.0f) > 0.0f || + fabsf(inverse_scale_f32 - 1.0f) > 0.0f) { + return 25; + } + if (expect_status(pernix_inverse_scale_f32(0.0f, &inverse_scale_f32), PERNIX_STATUS_INVALID_ARGUMENT) != 0 || + expect_status(pernix_compression_scale_f32(1.0f, 0, &compression_scale_f32), PERNIX_STATUS_INVALID_ARGUMENT) != 0 || + expect_status(pernix_decompression_scale_f32(1.0f, bit_width, 0), PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 26; + } + + double decompression_scale_f64 = 0.0; + double compression_scale_f64 = 0.0; + double inverse_scale_f64 = 0.0; + if (expect_status(pernix_decompression_scale_f64(127.0, bit_width, &decompression_scale_f64), PERNIX_STATUS_OK) != 0 || + expect_status(pernix_compression_scale_f64(127.0, bit_width, &compression_scale_f64), PERNIX_STATUS_OK) != 0 || + expect_status(pernix_inverse_scale_f64(decompression_scale_f64, &inverse_scale_f64), PERNIX_STATUS_OK) != 0) { + return 27; + } + if (fabs(decompression_scale_f64 - 1.0) > 0.0 || fabs(compression_scale_f64 - 1.0) > 0.0 || fabs(inverse_scale_f64 - 1.0) > 0.0) { + return 28; + } + + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, block_size, input_f32, 1.0f, compressed), + PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH) != 0) { + return 9; + } + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, 96, input_f32, 1.0f, compressed), + PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE) != 0) { + return 10; + } + if (expect_status(pernix_compress_block_f32((pernix_backend)255, bit_width, block_size, input_f32, 1.0f, compressed), + PERNIX_STATUS_UNSUPPORTED_BACKEND) != 0) { + return 11; + } + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0f, compressed), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 12; + } + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 1.0f, 0), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 13; + } + if (expect_status(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 0.0f, compressed), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 14; + } + if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0f, output_f32, true), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 15; + } + if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0f, 0, true), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 16; + } + if (expect_status(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, -1.0f, output_f32, true), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 17; + } + if (expect_status(pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f32, 1.0f, compressed, 0), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 18; + } + + if (expect_status(pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input_f64, 1.0, compressed), + PERNIX_STATUS_OK) != 0) { + return 19; + } + if (expect_status(pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0, output_f64, true), + PERNIX_STATUS_OK) != 0) { + return 20; + } + if (expect_status(pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, 0, 1.0, compressed, 1), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 21; + } + if (expect_status(pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 1.0, 0, 1, true), + PERNIX_STATUS_INVALID_ARGUMENT) != 0) { + return 22; + } + + return 0; +} diff --git a/tests/compression/avx2_compression_tests.cpp b/tests/compression/avx2_compression_tests.cpp deleted file mode 100644 index 1c2892b..0000000 --- a/tests/compression/avx2_compression_tests.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include <../../include/pernix/pernix.h> -#include - -#ifdef PERNIX_AVX2_ENABLED - -TYPED_TEST(CompressionTest, AVX2CompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm256_compress_block_avx2( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -TYPED_TEST(CompressionTest64, AVX2CompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm256_compress_block_avx2( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -#endif // PERNIX_AVX2_ENABLED diff --git a/tests/compression/avx512vbmi_compression_tests.cpp b/tests/compression/avx512vbmi_compression_tests.cpp deleted file mode 100644 index a6cb71d..0000000 --- a/tests/compression/avx512vbmi_compression_tests.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include - -#ifdef PERNIX_AVX512_VBMI_ENABLED - -TYPED_TEST(CompressionTest, AVX512VBMICompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm512_compress_block_avx512vbmi( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -TYPED_TEST(CompressionTest64, AVX512VBMICompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm512_compress_block_avx512vbmi( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -#endif // PERNIX_AVX512_VBMI_ENABLED diff --git a/tests/compression/bmi2_compression_tests.cpp b/tests/compression/bmi2_compression_tests.cpp deleted file mode 100644 index 85d3cac..0000000 --- a/tests/compression/bmi2_compression_tests.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include <../../include/pernix/pernix.h> -#include - -#ifdef PERNIX_BMI2_ENABLED - -TYPED_TEST(CompressionTest, BMI2CompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm256_compress_block_bmi2( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -TYPED_TEST(CompressionTest64, BMI2CompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::mm256_compress_block_bmi2( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - std::vector restored(this->testSet.elementsPerBlock); - pernix::decompress_block_fallback( - compressedData[block].data(), this->testSet.getScales()[block], restored.data()); - - expectDecompressedBlockNearSource(*this, restored, block); - } -} - -#endif // PERNIX_BMI2_ENABLED diff --git a/tests/compression/fallback_compression_tests.cpp b/tests/compression/fallback_compression_tests.cpp deleted file mode 100644 index 9b50109..0000000 --- a/tests/compression/fallback_compression_tests.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include <../../include/pernix/pernix.h> -#include - -TYPED_TEST(CompressionTest, FallbackCompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::compress_block_fallback( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectCompressedBlockEqualsReference(*this, compressedData[block], block); - } -} - -TYPED_TEST(CompressionTest64, FallbackCompressBlock) { - std::vector> compressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - compressedData[block].resize(TestFixture::BlockSize); - - pernix::compress_block_fallback( - this->testSet.getDecompressedData()[block].data(), 1 / this->testSet.getScales()[block], - reinterpret_cast(compressedData[block].data())); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectCompressedBlockEqualsReference(*this, compressedData[block], block); - } -} diff --git a/tests/decompression/avx2_decompression_tests.cpp b/tests/decompression/avx2_decompression_tests.cpp deleted file mode 100644 index e0f039f..0000000 --- a/tests/decompression/avx2_decompression_tests.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include <../../include/pernix/pernix.h> -#include - -#ifdef PERNIX_AVX2_ENABLED - -TYPED_TEST(DecompressionTest, AVX2DecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm256_decompress_block_avx2( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, AVX2DecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm256_decompress_block_avx2( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -#endif // PERNIX_AVX2_ENABLED diff --git a/tests/decompression/avx512vbmi_decompression_tests.cpp b/tests/decompression/avx512vbmi_decompression_tests.cpp deleted file mode 100644 index 446443a..0000000 --- a/tests/decompression/avx512vbmi_decompression_tests.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include <../../include/pernix/pernix.h> -#include - -#ifdef PERNIX_AVX512_VBMI_ENABLED - -TYPED_TEST(DecompressionTest, AVX512VBMIDecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm512_decompress_block_avx512vbmi( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, AVX512VBMIDecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm512_decompress_block_avx512vbmi( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -#endif // PERNIX_AVX512_VBMI_ENABLED diff --git a/tests/decompression/bmi2_decompression_tests.cpp b/tests/decompression/bmi2_decompression_tests.cpp deleted file mode 100644 index 11a8efb..0000000 --- a/tests/decompression/bmi2_decompression_tests.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include <../../include/pernix/pernix.h> -#include - -#ifdef PERNIX_BMI2_ENABLED - -TYPED_TEST(DecompressionTest, BMI2DecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm256_decompress_block_bmi2( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, BMI2DecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::mm256_decompress_block_bmi2( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -#endif // PERNIX_BMI2_ENABLED diff --git a/tests/decompression/fallback_decompression_tests.cpp b/tests/decompression/fallback_decompression_tests.cpp deleted file mode 100644 index 3c5dc1f..0000000 --- a/tests/decompression/fallback_decompression_tests.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include <../../include/pernix/pernix.h> -#include - -TYPED_TEST(DecompressionTest, FallbackDecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::decompress_block_fallback( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} - -TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { - std::vector> decompressedData(this->testSet.numberOfBlocks); - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - decompressedData[block].resize(this->testSet.elementsPerBlock); - - pernix::decompress_block_fallback( - this->testSet.getCompressedData()[block].data(), this->testSet.getScales()[block], decompressedData[block].data()); - } - - for (uint32_t block = 0; block < this->testSet.numberOfBlocks; block++) { - expectDecompressedBlockNearSource(*this, decompressedData[block], block); - } -} diff --git a/tests/deterministic_correctness_tests.cpp b/tests/deterministic_correctness_tests.cpp new file mode 100644 index 0000000..9dfabaf --- /dev/null +++ b/tests/deterministic_correctness_tests.cpp @@ -0,0 +1,404 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if PERNIX_TEST_BLOCK_SIZE == 64 +namespace { +constexpr u32 kBlockSize = 64; +constexpr u32 kBlocks = 3; + +enum class Pattern { + Zero, + Constant, + Increasing, + Decreasing, + SmallMagnitude, + LargeMagnitude, + QuantizationBoundary, + PseudoRandom +}; + +struct PatternCase { + Pattern pattern; + std::string_view name; +}; + +constexpr std::array kPatternCases{ + { + {Pattern::Zero, "zero"}, + {Pattern::Constant, "constant"}, + {Pattern::Increasing, "increasing"}, + {Pattern::Decreasing, "decreasing"}, + {Pattern::SmallMagnitude, "small_magnitude"}, + {Pattern::LargeMagnitude, "large_magnitude"}, + {Pattern::QuantizationBoundary, "quantization_boundary"}, + {Pattern::PseudoRandom, "pseudo_random"}, + } +}; + +constexpr std::array kComparisonBackends{ + { + PERNIX_BACKEND_X86_AVX2, PERNIX_BACKEND_X86_BMI2, + PERNIX_BACKEND_X86_AVX512_VBMI, PERNIX_BACKEND_ARM64_NEON, + PERNIX_BACKEND_ARM64_SVE, } +}; + +template +FloatT abs_value(const FloatT value) { + return value < FloatT{0} ? -value : value; +} + +template +FloatT max_abs(const std::vector& values) { + FloatT bmax = FloatT{0}; + for (const FloatT value : values) { + bmax = std::max(bmax, abs_value(value)); + } + return bmax; +} + +template +pernix_status compute_scale(const FloatT bmax, const u8 bit_width, FloatT& scale) { + if constexpr (std::is_same_v) { + return pernix_scale_f32(bmax, bit_width, &scale); + } else { + return pernix_scale_f64(bmax, bit_width, &scale); + } +} + +template +FloatT tolerance_for_value(const FloatT input, const FloatT scale) { + // Quantization contributes half a step in exact arithmetic. Reciprocal scaling and + // dequantization add magnitude-dependent rounding error, which is visible for f32 + // inputs whose magnitude approaches the number of exactly representable integers. + const FloatT epsilon = std::numeric_limits::epsilon(); + return (abs_value(scale) * static_cast(0.55)) + + (epsilon * ((abs_value(input) * static_cast(2)) + static_cast(64))); +} + +template +std::vector make_dataset(const Pattern pattern, const u8 bit_width, const u32 blocks = kBlocks) { + const u32 elements_per_block = pernix_elements_per_block(bit_width); + std::vector values(static_cast(elements_per_block) * blocks); + const bool one_bit = bit_width == 1; + const FloatT amplitude = [&] { + switch (pattern) { + case Pattern::SmallMagnitude: + return static_cast(1.0e-5); + case Pattern::LargeMagnitude: + return static_cast(1.0e6); + default: + return static_cast(16); + } + }(); + + std::mt19937 rng(0x5065726eU + bit_width); + std::uniform_real_distribution uniform(one_bit ? 0.0 : -1.0, 1.0); + + for (usize i = 0; i < values.size(); ++i) { + const FloatT unit = values.size() > 1 + ? static_cast((static_cast(i) / static_cast(values.size() - 1)) * 2.0 - 1.0) + : FloatT{0}; + switch (pattern) { + case Pattern::Zero: + values[i] = FloatT{0}; + break; + case Pattern::Constant: + values[i] = one_bit ? amplitude : amplitude / FloatT{2}; + break; + case Pattern::Increasing: + values[i] = one_bit ? static_cast(i % 2U) * amplitude : unit * amplitude; + break; + case Pattern::Decreasing: + values[i] = one_bit ? static_cast((i + 1U) % 2U) * amplitude : -unit * amplitude; + break; + case Pattern::SmallMagnitude: + case Pattern::LargeMagnitude: + values[i] = one_bit ? static_cast(i % 2U) * amplitude + : static_cast(((static_cast(i % 9U) - 4) / 4.0)) * amplitude; + break; + case Pattern::QuantizationBoundary: { + const FloatT step = amplitude / static_cast(17); + if (one_bit) { + values[i] = static_cast(i % 2U) * step; + } else { + const int bucket = static_cast(i % 9U) - 4; + const FloatT nudge = (i % 2U == 0U) ? step * static_cast(0.49) : step * static_cast(-0.49); + values[i] = (static_cast(bucket) * step) + nudge; + } + break; + } + case Pattern::PseudoRandom: + values[i] = static_cast(uniform(rng)) * amplitude; + break; + } + } + + return values; +} + +template +void expect_near_input(const std::vector& input, const std::vector& restored, const FloatT scale) { + ASSERT_EQ(restored.size(), input.size()); + for (usize i = 0; i < input.size(); ++i) { + const FloatT tolerance = tolerance_for_value(input[i], scale); + ASSERT_NEAR(restored[i], input[i], tolerance) << "element=" << i << ", scale=" << scale; + } +} + +template +pernix_status compress_blocks(pernix_backend backend, const u8 bit_width, const std::vector& input, const FloatT inverse_scale, + std::vector& compressed, const u32 blocks) { + if constexpr (std::is_same_v) { + return pernix_compress_blocks_f32(backend, bit_width, kBlockSize, input.data(), inverse_scale, compressed.data(), blocks); + } else { + return pernix_compress_blocks_f64(backend, bit_width, kBlockSize, input.data(), inverse_scale, compressed.data(), blocks); + } +} + +template +pernix_status decompress_blocks(pernix_backend backend, const u8 bit_width, const std::vector& compressed, const FloatT scale, + std::vector& output, const u32 blocks, const bool sign_values = true) { + if constexpr (std::is_same_v) { + return pernix_decompress_blocks_f32(backend, bit_width, kBlockSize, compressed.data(), scale, output.data(), blocks, sign_values); + } else { + return pernix_decompress_blocks_f64(backend, bit_width, kBlockSize, compressed.data(), scale, output.data(), blocks, sign_values); + } +} + +template +void expect_fallback_round_trip(const u8 bit_width, const PatternCase& pattern_case) { + SCOPED_TRACE(::testing::Message() << "bit_width=" << static_cast(bit_width) << ", pattern=" << pattern_case.name); + + const auto input = make_dataset(pattern_case.pattern, bit_width); + FloatT scale = FloatT{0}; + ASSERT_EQ(compute_scale(max_abs(input), bit_width, scale), PERNIX_STATUS_OK); + + std::vector compressed(kBlocks * kBlockSize); + std::vector restored(input.size(), FloatT{}); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input, FloatT{1} / scale, compressed, kBlocks), PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed, scale, restored, kBlocks), PERNIX_STATUS_OK); + expect_near_input(input, restored, scale); +} + +template +void expect_backend_matches_fallback(const pernix_backend backend, const u8 bit_width) { + SCOPED_TRACE(::testing::Message() << "backend=" << static_cast(backend) << ", bit_width=" << static_cast(bit_width)); + + const auto input = make_dataset(Pattern::PseudoRandom, bit_width, 2); + FloatT scale = FloatT{0}; + ASSERT_EQ(compute_scale(max_abs(input), bit_width, scale), PERNIX_STATUS_OK); + + std::vector fallback_compressed(2 * kBlockSize); + std::vector backend_compressed(2 * kBlockSize); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input, FloatT{1} / scale, fallback_compressed, 2), PERNIX_STATUS_OK); + + const auto compress_status = compress_blocks(backend, bit_width, input, FloatT{1} / scale, backend_compressed, 2); + if (compress_status != PERNIX_STATUS_OK) { + EXPECT_TRUE(compress_status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || compress_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || + compress_status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + return; + } + + std::vector fallback_restored(input.size(), FloatT{}); + std::vector backend_restored(input.size(), FloatT{}); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, fallback_compressed, scale, fallback_restored, 2), PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, backend_compressed, scale, backend_restored, 2), PERNIX_STATUS_OK); + + const FloatT tolerance = + (abs_value(scale) * static_cast(1.1)) + (std::numeric_limits::epsilon() * static_cast(64)); + for (usize i = 0; i < input.size(); ++i) { + ASSERT_NEAR(backend_restored[i], fallback_restored[i], tolerance) << "element=" << i << ", scale=" << scale; + } + + std::vector backend_decompressed(input.size(), FloatT{}); + const auto decompress_status = decompress_blocks(backend, bit_width, fallback_compressed, scale, backend_decompressed, 2); + if (decompress_status != PERNIX_STATUS_OK) { + EXPECT_TRUE(decompress_status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || + decompress_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || decompress_status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + return; + } + for (usize i = 0; i < input.size(); ++i) { + ASSERT_NEAR(backend_decompressed[i], fallback_restored[i], tolerance) << "element=" << i << ", scale=" << scale; + } +} +} // namespace + +TEST(DeterministicCorrectnessTest, F32FallbackRoundTripsAllPatternsAndBitWidths) { + for (u8 bit_width = pernix_min_bit_width(); bit_width <= pernix_max_bit_width(); ++bit_width) { + for (const PatternCase& pattern_case : kPatternCases) { + expect_fallback_round_trip(bit_width, pattern_case); + } + } +} + +TEST(DeterministicCorrectnessTest, F64FallbackRoundTripsAllPatternsAndBitWidths) { + for (u8 bit_width = pernix_min_bit_width(); bit_width <= pernix_max_bit_width(); ++bit_width) { + for (const PatternCase& pattern_case : kPatternCases) { + expect_fallback_round_trip(bit_width, pattern_case); + } + } +} + +TEST(DeterministicCorrectnessTest, SimdBackendsMatchFallbackWhereAvailable) { + for (const pernix_backend backend : kComparisonBackends) { + for (u8 bit_width = pernix_min_bit_width(); bit_width <= pernix_max_bit_width(); ++bit_width) { + expect_backend_matches_fallback(backend, bit_width); + expect_backend_matches_fallback(backend, bit_width); + } + } +} + +TEST(DeterministicCorrectnessTest, F32AndF64HaveConsistentFallbackSemantics) { + constexpr u8 bit_width = 12; + const u32 elements = pernix_elements_per_block(bit_width) * 2U; + std::vector input_f32(elements); + for (usize i = 0; i < input_f32.size(); ++i) { + input_f32[i] = static_cast(static_cast(i % 9U) - 4); + } + std::vector input_f64(input_f32.begin(), input_f32.end()); + + std::vector compressed_f32(2 * kBlockSize); + std::vector compressed_f64(2 * kBlockSize); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input_f32, 1.0f, compressed_f32, 2), PERNIX_STATUS_OK); + ASSERT_EQ(compress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, input_f64, 1.0, compressed_f64, 2), PERNIX_STATUS_OK); + EXPECT_EQ(compressed_f64, compressed_f32); + + std::vector restored_f32(input_f32.size()); + std::vector restored_f64(input_f64.size()); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed_f32, 1.0f, restored_f32, 2), PERNIX_STATUS_OK); + ASSERT_EQ(decompress_blocks(PERNIX_BACKEND_FALLBACK, bit_width, compressed_f64, 1.0, restored_f64, 2), PERNIX_STATUS_OK); + + for (usize i = 0; i < restored_f32.size(); ++i) { + EXPECT_DOUBLE_EQ(static_cast(restored_f32[i]), restored_f64[i]); + } +} + +TEST(DeterministicCorrectnessTest, SignValuesFalseUsesUnsignedPackedValuesForAllBitWidths) { + for (u8 bit_width = pernix_min_bit_width(); bit_width <= pernix_max_bit_width(); ++bit_width) { + SCOPED_TRACE(::testing::Message() << "bit_width=" << static_cast(bit_width)); + std::vector packed(kBlockSize, 0xFFU); + const u32 elements = pernix_elements_per_block(bit_width); + std::vector signed_values(elements); + std::vector unsigned_values(elements); + std::vector avx512_unsigned_values(elements); + + ASSERT_EQ( + pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, signed_values.data(), true), + PERNIX_STATUS_OK); + ASSERT_EQ( + pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, unsigned_values.data(), false), + PERNIX_STATUS_OK); + + const float expected_unsigned = static_cast((1U << bit_width) - 1U); + EXPECT_TRUE(std::ranges::all_of(unsigned_values, [expected_unsigned](const float value) { return value == expected_unsigned; })); + + const pernix_status avx512_status = pernix_decompress_block_f32(PERNIX_BACKEND_X86_AVX512_VBMI, bit_width, kBlockSize, + packed.data(), 1.0f, avx512_unsigned_values.data(), false); + if (avx512_status == PERNIX_STATUS_OK) { + EXPECT_EQ(avx512_unsigned_values, unsigned_values); + } else { + EXPECT_TRUE(avx512_status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || avx512_status == PERNIX_STATUS_UNSUPPORTED_BACKEND || + avx512_status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + } + + if (bit_width == 1) { + EXPECT_EQ(signed_values[0], 1.0f); + } else { + EXPECT_EQ(signed_values[0], -1.0f); + } + } +} + +TEST(DeterministicCorrectnessTest, Avx512TableDrivenSmallWidthsMatchFallbackForMixedBits) { + constexpr std::array bit_widths{1, 2, 4}; + std::vector packed(kBlockSize); + for (usize i = 0; i < packed.size(); ++i) { + packed[i] = static_cast(i * 37U + 0x5aU); + } + + for (const u8 bit_width : bit_widths) { + SCOPED_TRACE(::testing::Message() << "bit_width=" << static_cast(bit_width)); + const u32 elements = pernix_elements_per_block(bit_width); + + for (const bool sign_values : {false, true}) { + std::vector fallback_output(elements); + std::vector avx512_output(elements); + + ASSERT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, packed.data(), 1.0f, + fallback_output.data(), sign_values), + PERNIX_STATUS_OK); + + const pernix_status status = pernix_decompress_block_f32(PERNIX_BACKEND_X86_AVX512_VBMI, bit_width, kBlockSize, packed.data(), + 1.0f, avx512_output.data(), sign_values); + if (status != PERNIX_STATUS_OK) { + EXPECT_TRUE(status == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION || status == PERNIX_STATUS_UNSUPPORTED_BACKEND || + status == PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + GTEST_SKIP(); + } + + EXPECT_EQ(avx512_output, fallback_output); + } + } +} + +TEST(DeterministicCorrectnessTest, CppSpanWrappersMatchCApiOutputAndStatus) { + constexpr u8 bit_width = 10; + const auto input = make_dataset(Pattern::Increasing, bit_width, 1); + float scale = 0.0f; + ASSERT_EQ(compute_scale(max_abs(input), bit_width, scale), PERNIX_STATUS_OK); + + std::vector c_compressed(kBlockSize); + std::vector cpp_compressed(kBlockSize); + EXPECT_EQ(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, input.data(), 1.0f / scale, c_compressed.data()), + PERNIX_STATUS_OK); + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), 1.0f / scale, + std::span(cpp_compressed)), + PERNIX_STATUS_OK); + EXPECT_EQ(cpp_compressed, c_compressed); + + std::vector c_restored(input.size()); + std::vector cpp_restored(input.size()); + EXPECT_EQ( + pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, kBlockSize, c_compressed.data(), scale, c_restored.data(), true), + PERNIX_STATUS_OK); + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(cpp_compressed), scale, + std::span(cpp_restored)), + PERNIX_STATUS_OK); + EXPECT_EQ(cpp_restored, c_restored); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, 0, kBlockSize, std::span(input), 1.0f / scale, + std::span(cpp_compressed)), + pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, kBlockSize, input.data(), 1.0f / scale, c_compressed.data())); +} + +TEST(DeterministicCorrectnessTest, SpanValidationRejectsUndersizedMultiBlockBuffers) { + constexpr u8 bit_width = 8; + const u32 elements = pernix_elements_per_block(bit_width) * 2U; + std::vector input(elements, 1.0f); + std::vector compressed(2U * kBlockSize); + std::vector restored(elements); + + EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, + std::span(input).first(input.size() - 1U), 1.0f, std::span(compressed), 2), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), 1.0f, + std::span(compressed).first(compressed.size() - 1U), 2), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, + std::span(compressed).first(compressed.size() - 1U), 1.0f, std::span(restored), 2), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(compressed), 1.0f, + std::span(restored).first(restored.size() - 1U), 2), + PERNIX_STATUS_INVALID_ARGUMENT); +} +#endif diff --git a/tests/fallback_edge_tests.cpp b/tests/fallback_edge_tests.cpp deleted file mode 100644 index 2bdee5a..0000000 --- a/tests/fallback_edge_tests.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include -#include - -#include - -#include -#include -#include -#include - -TEST(FallbackDecompressionEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { - const std::array input{0x08}; - std::array output{}; - - ASSERT_EQ(pernix::decompress_block_fallback<4>(input.data(), 1.0F, output.data()), 0); - - EXPECT_EQ(output[0], -8.0F); -} - -TEST(FallbackCompressionEdgeTest, ClearsUnusedPaddingBytes) { - std::array input{}; - std::array output{}; - output.fill(0xA5); - - ASSERT_EQ(pernix::compress_block_fallback<24>(input.data(), 1.0F, output.data()), 0); - - EXPECT_EQ(output[63], 0); -} - -TEST(FallbackCompressionEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { - std::array input{}; - input[0] = std::numeric_limits::infinity(); - input[1] = -std::numeric_limits::infinity(); - input[2] = std::numeric_limits::quiet_NaN(); - std::array compressed{}; - std::array restored{}; - - ASSERT_EQ(pernix::compress_block_fallback<4>(input.data(), 1.0F, compressed.data()), 0); - ASSERT_EQ(pernix::decompress_block_fallback<4>(compressed.data(), 1.0F, restored.data()), 0); - - EXPECT_EQ(restored[0], 7.0F); - EXPECT_EQ(restored[1], -8.0F); - EXPECT_EQ(restored[2], 0.0F); -} diff --git a/tests/fallback_tests.cpp b/tests/fallback_tests.cpp new file mode 100644 index 0000000..5f5f445 --- /dev/null +++ b/tests/fallback_tests.cpp @@ -0,0 +1,1016 @@ +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) && defined(__aarch64__) +#include +#ifndef HWCAP_SVE +#define HWCAP_SVE (1 << 22) +#endif +#ifndef HWCAP2_SVE2 +#define HWCAP2_SVE2 (1 << 1) +#endif +#endif + +namespace { +template +void expect_backend_compressed_block_matches_reference(FixtureT& fixture, const pernix_backend backend, CompressFn compress_fn) { + using ScaleType = std::remove_cvref_t; + std::vector > compressed(fixture.testSet.numberOfBlocks); + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { + compressed[b].resize(FixtureT::BlockSize); + const auto status = compress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[b].data(), + ScaleType{1} / fixture.testSet.getScales()[b], compressed[b].data()); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { + expectCompressedBlockEqualsReference(fixture, compressed[b], b); + } +} + +template +void expect_backend_decompressed_block_matches_source(FixtureT& fixture, const pernix_backend backend, DecompressFn decompress_fn) { + using ValueType = std::remove_cvref_t; + std::vector > decompressed(fixture.testSet.numberOfBlocks); + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { + decompressed[b].resize(fixture.testSet.elementsPerBlock); + const auto status = decompress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed[b].data(), true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; ++b) { + expectDecompressedBlockNearSource(fixture, decompressed[b], b); + } +} + +template +void expect_backend_compress_blocks_matches_scalar(FixtureT& fixture, CompressFn compress_fn, const pernix_backend backend) { + using ScaleType = std::remove_cvref_t; + + const u32 blocks = fixture.testSet.numberOfBlocks; + const u32 elements_per_block = fixture.testSet.elementsPerBlock; + const u32 total_elements = blocks * elements_per_block; + const usize compressed_bytes = static_cast(blocks) * FixtureT::BlockSize; + std::vector input(total_elements); + + for (u32 block = 0; block < blocks; ++block) { + std::copy_n(fixture.testSet.getDecompressedData()[block].data(), elements_per_block, + input.data() + static_cast(block) * elements_per_block); + } + + const ScaleType scale_inv = ScaleType{1} / fixture.testSet.getScales()[0]; + std::vector scalar_output(compressed_bytes); + std::vector stdpar_output(compressed_bytes); + + auto status = compress_fn(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, input.data(), scale_inv, + scalar_output.data(), blocks); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + status = compress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, input.data(), scale_inv, stdpar_output.data(), blocks); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_output, scalar_output); +} + +template +void expect_backend_decompress_blocks_matches_scalar(FixtureT& fixture, DecompressFn decompress_fn, const pernix_backend backend, + const bool sign_values = true) { + using ValueType = std::remove_cvref_t; + + const u32 blocks = fixture.testSet.numberOfBlocks; + const u32 elements_per_block = fixture.testSet.elementsPerBlock; + const u32 total_elements = blocks * elements_per_block; + std::vector compressed(static_cast(blocks) * FixtureT::BlockSize); + + for (u32 block = 0; block < blocks; ++block) { + std::copy_n(fixture.testSet.getCompressedData()[block].data(), FixtureT::BlockSize, + compressed.data() + static_cast(block) * FixtureT::BlockSize); + } + + std::vector scalar_output(total_elements); + std::vector stdpar_output(total_elements); + const auto scale = fixture.testSet.getScales()[0]; + + auto status = decompress_fn(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, compressed.data(), scale, + scalar_output.data(), blocks, sign_values); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + status = decompress_fn(backend, FixtureT::BitWidth, FixtureT::BlockSize, compressed.data(), scale, stdpar_output.data(), blocks, + sign_values); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_output, scalar_output); +} +} // namespace + +// --------------------------------------------------------------------------- +// Fallback compress: verify byte-exact match against the reference +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, FallbackCompressBlock) { + std::vector > compressed(this->testSet.numberOfBlocks); + + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { + compressed[b].resize(TestFixture::BlockSize); + + const auto status = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getDecompressedData()[b].data(), 1.0f / this->testSet.getScales()[b], + compressed[b].data()); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { + expectCompressedBlockEqualsReference(*this, compressed[b], b); + } +} + +TYPED_TEST(CompressionTest64, FallbackCompressBlock) { + std::vector > compressed(this->testSet.numberOfBlocks); + + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { + compressed[b].resize(TestFixture::BlockSize); + + const auto status = pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getDecompressedData()[b].data(), 1.0 / this->testSet.getScales()[b], + compressed[b].data()); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { + expectCompressedBlockEqualsReference(*this, compressed[b], b); + } +} + +// --------------------------------------------------------------------------- +// Fallback decompress: verify near-source match +// --------------------------------------------------------------------------- + +TYPED_TEST(DecompressionTest, FallbackDecompressBlock) { + std::vector > decompressed(this->testSet.numberOfBlocks); + + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { + decompressed[b].resize(this->testSet.elementsPerBlock); + + const auto status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getCompressedData()[b].data(), this->testSet.getScales()[b], + decompressed[b].data(), true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { + expectDecompressedBlockNearSource(*this, decompressed[b], b); + } +} + +TYPED_TEST(DecompressionTest64, FallbackDecompressBlock) { + std::vector > decompressed(this->testSet.numberOfBlocks); + + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { + decompressed[b].resize(this->testSet.elementsPerBlock); + + const auto status = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, + this->testSet.getCompressedData()[b].data(), this->testSet.getScales()[b], + decompressed[b].data(), true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + } + + for (u32 b = 0; b < this->testSet.numberOfBlocks; b++) { + expectDecompressedBlockNearSource(*this, decompressed[b], b); + } +} + +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TYPED_TEST(CompressionTest, FallbackStdparCompressBlock) { + expect_backend_compressed_block_matches_reference(*this, PERNIX_BACKEND_FALLBACK_STDPAR, &pernix_compress_block_f32); +} + +TYPED_TEST(CompressionTest64, FallbackStdparCompressBlock) { + expect_backend_compressed_block_matches_reference(*this, PERNIX_BACKEND_FALLBACK_STDPAR, &pernix_compress_block_f64); +} + +TYPED_TEST(DecompressionTest, FallbackStdparDecompressBlock) { + expect_backend_decompressed_block_matches_source(*this, PERNIX_BACKEND_FALLBACK_STDPAR, &pernix_decompress_block_f32); +} + +TYPED_TEST(DecompressionTest64, FallbackStdparDecompressBlock) { + expect_backend_decompressed_block_matches_source(*this, PERNIX_BACKEND_FALLBACK_STDPAR, &pernix_decompress_block_f64); +} + +TYPED_TEST(CompressionTest, FallbackStdparCompressBlocksMatchesScalar) { + expect_backend_compress_blocks_matches_scalar(*this, &pernix_compress_blocks_f32, PERNIX_BACKEND_FALLBACK_STDPAR); +} + +TYPED_TEST(CompressionTest64, FallbackStdparCompressBlocksMatchesScalar) { + expect_backend_compress_blocks_matches_scalar(*this, &pernix_compress_blocks_f64, PERNIX_BACKEND_FALLBACK_STDPAR); +} + +TYPED_TEST(DecompressionTest, FallbackStdparDecompressBlocksMatchesScalar) { + expect_backend_decompress_blocks_matches_scalar(*this, &pernix_decompress_blocks_f32, PERNIX_BACKEND_FALLBACK_STDPAR); +} + +TYPED_TEST(DecompressionTest64, FallbackStdparDecompressBlocksMatchesScalar) { + expect_backend_decompress_blocks_matches_scalar(*this, &pernix_decompress_blocks_f64, PERNIX_BACKEND_FALLBACK_STDPAR); +} +#endif + +// --------------------------------------------------------------------------- +// Multi-block roundtrip via compress_blocks / decompress_blocks (fallback) +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, FallbackCompressBlocksRoundtrip) { + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; + const u32 total = nb * epb; + + std::vector flat(total); + for (u32 b = 0; b < nb; b++) { + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); + } + + // Compute a single scale that covers all blocks + float max_abs = 0.0f; + for (u32 i = 0; i < total; i++) { + max_abs = std::max(max_abs, std::abs(flat[i])); + } + const float q = static_cast(decltype(this->testSet)::quantization_levels); + const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); + const float scale_inv = 1.0f / scale; + + std::vector compressed(nb * TestFixture::BlockSize); + auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), scale_inv, + compressed.data(), nb); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + std::vector restored(total); + status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, + restored.data(), nb, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + const float tol = (std::abs(scale) * 0.5f) + (std::numeric_limits::epsilon() * 16.0f); + for (u32 i = 0; i < total; i++) { + EXPECT_NEAR(restored[i], flat[i], tol); + } +} + +TYPED_TEST(CompressionTest64, FallbackCompressBlocksRoundtrip) { + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; + const u32 total = nb * epb; + + std::vector flat(total); + for (u32 b = 0; b < nb; b++) { + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); + } + + // Compute a single scale that covers all blocks + double max_abs = 0.0; + for (u32 i = 0; i < total; i++) { + max_abs = std::max(max_abs, std::abs(flat[i])); + } + const double q = static_cast(decltype(this->testSet)::quantization_levels); + const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); + const double scale_inv = 1.0 / scale; + + std::vector compressed(nb * TestFixture::BlockSize); + auto status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), scale_inv, + compressed.data(), nb); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + std::vector restored(total); + status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, + restored.data(), nb, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + const double tol = (std::abs(scale) * 0.5) + (std::numeric_limits::epsilon() * 16.0); + for (u32 i = 0; i < total; i++) { + EXPECT_NEAR(restored[i], flat[i], tol); + } +} + +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TYPED_TEST(CompressionTest, FallbackStdparCompressBlocksRoundtrip) { + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; + const u32 total = nb * epb; + + std::vector flat(total); + for (u32 b = 0; b < nb; ++b) { + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); + } + + float max_abs = 0.0f; + for (u32 i = 0; i < total; ++i) { + max_abs = std::max(max_abs, std::abs(flat[i])); + } + const float q = static_cast(decltype(this->testSet)::quantization_levels); + const float scale = (max_abs > 0.0f && q > 0.0f) ? (max_abs / q) : std::numeric_limits::epsilon(); + const float scale_inv = 1.0f / scale; + + std::vector compressed(nb * TestFixture::BlockSize); + auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), + scale_inv, compressed.data(), nb); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + std::vector restored(total); + status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), + scale, restored.data(), nb, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + const float tol = (std::abs(scale) * 0.5f) + (std::numeric_limits::epsilon() * 16.0f); + for (u32 i = 0; i < total; ++i) { + EXPECT_NEAR(restored[i], flat[i], tol); + } +} + +TYPED_TEST(CompressionTest64, FallbackStdparCompressBlocksRoundtrip) { + const u32 nb = this->testSet.numberOfBlocks; + const u32 epb = this->testSet.elementsPerBlock; + const u32 total = nb * epb; + + std::vector flat(total); + for (u32 b = 0; b < nb; ++b) { + std::copy_n(this->testSet.getDecompressedData()[b].data(), epb, flat.data() + b * epb); + } + + double max_abs = 0.0; + for (u32 i = 0; i < total; ++i) { + max_abs = std::max(max_abs, std::abs(flat[i])); + } + const double q = static_cast(decltype(this->testSet)::quantization_levels); + const double scale = (max_abs > 0.0 && q > 0.0) ? (max_abs / q) : std::numeric_limits::epsilon(); + const double scale_inv = 1.0 / scale; + + std::vector compressed(nb * TestFixture::BlockSize); + auto status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, flat.data(), + scale_inv, compressed.data(), nb); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + std::vector restored(total); + status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), + scale, restored.data(), nb, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + const double tol = (std::abs(scale) * 0.5) + (std::numeric_limits::epsilon() * 16.0); + for (u32 i = 0; i < total; ++i) { + EXPECT_NEAR(restored[i], flat[i], tol); + } +} +#endif + +// --------------------------------------------------------------------------- +// blocks API with a single block should match the block API exactly +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, SingleBlockCompressBlocksMatchesBlock) { + const auto& src = this->testSet.getDecompressedData()[0]; + const float scale_inv = 1.0f / this->testSet.getScales()[0]; + + std::vector blockOut(TestFixture::BlockSize); + std::vector blocksOut(TestFixture::BlockSize); + + auto s1 = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, src.data(), scale_inv, + blockOut.data()); + ASSERT_EQ(s1, PERNIX_STATUS_OK); + + auto s2 = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, src.data(), scale_inv, + blocksOut.data(), 1); + ASSERT_EQ(s2, PERNIX_STATUS_OK); + + for (u32 i = 0; i < TestFixture::BlockSize; i++) { + EXPECT_EQ(blockOut[i], blocksOut[i]) << "byte " << i; + } +} + +TYPED_TEST(DecompressionTest, SingleBlockDecompressBlocksMatchesBlock) { + const auto& compressed = this->testSet.getCompressedData()[0]; + const float scale = this->testSet.getScales()[0]; + const u32 epb = this->testSet.elementsPerBlock; + + std::vector blockOut(epb); + std::vector blocksOut(epb); + + auto s1 = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, + blockOut.data(), true); + ASSERT_EQ(s1, PERNIX_STATUS_OK); + + auto s2 = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), scale, + blocksOut.data(), 1, true); + ASSERT_EQ(s2, PERNIX_STATUS_OK); + + for (u32 i = 0; i < epb; i++) { + EXPECT_FLOAT_EQ(blockOut[i], blocksOut[i]) << "element " << i; + } +} + +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TYPED_TEST(CompressionTest, SingleBlockStdparCompressBlocksMatchesBlock) { + const auto& src = this->testSet.getDecompressedData()[0]; + const float scale_inv = 1.0f / this->testSet.getScales()[0]; + + std::vector block_out(TestFixture::BlockSize); + std::vector blocks_out(TestFixture::BlockSize); + + auto s1 = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, src.data(), + scale_inv, block_out.data()); + ASSERT_EQ(s1, PERNIX_STATUS_OK); + + auto s2 = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, src.data(), + scale_inv, blocks_out.data(), 1); + ASSERT_EQ(s2, PERNIX_STATUS_OK); + + for (u32 i = 0; i < TestFixture::BlockSize; ++i) { + EXPECT_EQ(block_out[i], blocks_out[i]) << "byte " << i; + } +} + +TYPED_TEST(DecompressionTest, SingleBlockStdparDecompressBlocksMatchesBlock) { + const auto& compressed = this->testSet.getCompressedData()[0]; + const float scale = this->testSet.getScales()[0]; + const u32 epb = this->testSet.elementsPerBlock; + + std::vector block_out(epb); + std::vector blocks_out(epb); + + auto s1 = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), + scale, block_out.data(), true); + ASSERT_EQ(s1, PERNIX_STATUS_OK); + + auto s2 = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, TestFixture::BitWidth, TestFixture::BlockSize, compressed.data(), + scale, blocks_out.data(), 1, true); + ASSERT_EQ(s2, PERNIX_STATUS_OK); + + for (u32 i = 0; i < epb; ++i) { + EXPECT_FLOAT_EQ(block_out[i], blocks_out[i]) << "element " << i; + } +} +#endif + +// --------------------------------------------------------------------------- +// Edge-case behavioural tests (fallback, block_size=64) +// --------------------------------------------------------------------------- + +TEST(FallbackEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { + constexpr u32 BS = 64; + const std::array input{0x08}; + + pernix_status st; + std::array output{}; + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, output.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + EXPECT_EQ(output[0], -8.0f); +} + +TEST(FallbackEdgeTest, ClearsUnusedPaddingBytes) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + std::array output{}; + output.fill(0xA5); + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, output.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + EXPECT_EQ(output[BS - 1], 0); +} + +TEST(FallbackEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { + constexpr u32 BS = 64; + constexpr u32 BW = 4; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + input[0] = std::numeric_limits::infinity(); + input[1] = -std::numeric_limits::infinity(); + input[2] = std::numeric_limits::quiet_NaN(); + + std::array compressed{}; + std::array restored{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), 1.0f, restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(restored[0], 7.0f); + EXPECT_EQ(restored[1], -8.0f); + EXPECT_EQ(restored[2], 0.0f); +} + +TEST(FallbackEdgeTest, SignValuesFalseTreatsPackedValuesAsUnsigned) { + constexpr u32 BS = 64; + const std::array input{0x0F}; + + std::array signed_output{}; + std::array unsigned_output{}; + + auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, signed_output.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, unsigned_output.data(), false); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(signed_output[0], -1.0f); + EXPECT_EQ(unsigned_output[0], 15.0f); +} + +TEST(FallbackEdgeTest, FixedBlockInvariantAndMixedSignMultiBlockRoundTrip) { + constexpr u32 BS = 64; + constexpr u32 BW = 12; + constexpr u32 blocks = 3; + constexpr u32 EPB = (BS * 8) / BW; + constexpr u32 total = EPB * blocks; + + static_assert(BS == 64); + static_assert(EPB == 42); + EXPECT_EQ(pernix_compressed_block_size(), BS); + EXPECT_EQ(pernix_elements_per_block(BW), EPB); + + std::array input_f32{}; + std::array input_f64{}; + for (u32 i = 0; i < total; ++i) { + const auto centered = static_cast(i % 31U) - 15; + input_f32[i] = static_cast(centered) * 0.25f; + input_f64[i] = static_cast(centered) * 0.25; + } + + std::array compressed_f32{}; + std::array compressed_f64{}; + std::array restored_f32{}; + std::array restored_f64{}; + + auto status = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input_f32.data(), 4.0f, compressed_f32.data(), blocks); + ASSERT_EQ(status, PERNIX_STATUS_OK); + status = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed_f32.data(), 0.25f, restored_f32.data(), blocks, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + status = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input_f64.data(), 4.0, compressed_f64.data(), blocks); + ASSERT_EQ(status, PERNIX_STATUS_OK); + status = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed_f64.data(), 0.25, restored_f64.data(), blocks, true); + ASSERT_EQ(status, PERNIX_STATUS_OK); + + for (u32 i = 0; i < total; ++i) { + EXPECT_FLOAT_EQ(restored_f32[i], input_f32[i]) << "f32 element " << i; + EXPECT_DOUBLE_EQ(restored_f64[i], input_f64[i]) << "f64 element " << i; + } + + std::array unsigned_output{}; + status = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, compressed_f32.data(), 1.0f, unsigned_output.data(), false); + ASSERT_EQ(status, PERNIX_STATUS_OK); +} + +TEST(FallbackEdgeTest, CApiRejectsInvalidScale) { + constexpr u32 BS = 64; + constexpr u32 BW = 8; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + std::array compressed{}; + std::array output{}; + + EXPECT_EQ(pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 0.0f, compressed.data()), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), -1.0f, output.data(), true), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ( + pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), std::numeric_limits::infinity(), compressed.data()), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(FallbackEdgeTest, CApiRejectsInvalidScaleF64) { + constexpr u32 BS = 64; + constexpr u32 BW = 8; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + std::array compressed{}; + std::array output{}; + + EXPECT_EQ(pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 0.0, compressed.data()), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), -1.0, output.data(), true), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(FallbackEdgeTest, ScaleHelpersValidateInputs) { + float scale_f32 = 0.0f; + double scale_f64 = 0.0; + + EXPECT_EQ(pernix_scale_f32(32767.0f, 16, &scale_f32), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(scale_f32, 1.0f); + EXPECT_EQ(pernix_scale_f64(32767.0, 16, &scale_f64), PERNIX_STATUS_OK); + EXPECT_DOUBLE_EQ(scale_f64, 1.0); + EXPECT_EQ(pernix_scale_f32(1.0f, 0, &scale_f32), PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_scale_f32(-1.0f, 16, &scale_f32), PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_scale_f64(std::numeric_limits::infinity(), 16, &scale_f64), PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix_scale_f64(1.0, 16, nullptr), PERNIX_STATUS_INVALID_ARGUMENT); +} + +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TEST(FallbackStdparEdgeTest, SignExtensionIsWellDefinedForNegativeValues) { + constexpr u32 BS = 64; + const std::array input{0x08}; + + std::array scalar_output{}; + std::array output{}; + auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, scalar_output.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, 4, BS, input.data(), 1.0f, output.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + EXPECT_EQ(output, scalar_output); + EXPECT_EQ(output[0], -8.0f); +} + +TEST(FallbackStdparEdgeTest, ClearsUnusedPaddingBytes) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + std::array output{}; + output.fill(0xA5); + + const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, output.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + EXPECT_EQ(output[BS - 1], 0); +} + +TEST(FallbackStdparEdgeTest, ClampsNonFiniteAndOutOfRangeBeforeNarrowing) { + constexpr u32 BS = 64; + constexpr u32 BW = 4; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + input[0] = std::numeric_limits::infinity(); + input[1] = -std::numeric_limits::infinity(); + input[2] = std::numeric_limits::quiet_NaN(); + + std::array scalar_compressed{}; + std::array stdpar_compressed{}; + std::array scalar_restored{}; + std::array stdpar_restored{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, scalar_compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, stdpar_compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, scalar_compressed.data(), 1.0f, scalar_restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, stdpar_compressed.data(), 1.0f, stdpar_restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_compressed, scalar_compressed); + EXPECT_EQ(stdpar_restored, scalar_restored); + EXPECT_EQ(stdpar_restored[0], 7.0f); + EXPECT_EQ(stdpar_restored[1], -8.0f); + EXPECT_EQ(stdpar_restored[2], 0.0f); +} + +TEST(FallbackStdparEdgeTest, SignValuesFalseMatchesScalar) { + constexpr u32 BS = 64; + const std::array input{0x0F}; + + std::array scalar_output{}; + std::array stdpar_output{}; + + auto st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 4, BS, input.data(), 1.0f, scalar_output.data(), false); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, 4, BS, input.data(), 1.0f, stdpar_output.data(), false); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_output, scalar_output); + EXPECT_EQ(stdpar_output[0], 15.0f); +} + +TEST(FallbackStdparEdgeTest, RoundingAndClampingMatchScalar) { + constexpr u32 BS = 64; + constexpr u32 BW = 4; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + input[0] = 0.49f; + input[1] = 0.50f; + input[2] = 1.49f; + input[3] = 1.50f; + input[4] = 7.49f; + input[5] = 7.50f; + input[6] = -7.49f; + input[7] = -7.50f; + input[8] = 99.0f; + input[9] = -99.0f; + + std::array scalar_compressed{}; + std::array stdpar_compressed{}; + std::array scalar_restored{}; + std::array stdpar_restored{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, scalar_compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, stdpar_compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, scalar_compressed.data(), 1.0f, scalar_restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, stdpar_compressed.data(), 1.0f, stdpar_restored.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(stdpar_compressed, scalar_compressed); + EXPECT_EQ(stdpar_restored, scalar_restored); +} + +TEST(FallbackStdparEdgeTest, RepeatedExecutionIsDeterministic) { + constexpr u32 BS = 64; + constexpr u32 BW = 12; + constexpr u32 blocks = 3; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < input.size(); ++i) { + input[i] = static_cast((static_cast(i % 23U) - 11) * 0.25f); + } + + std::array first_compressed{}; + std::array second_compressed{}; + std::array first_restored{}; + std::array second_restored{}; + + auto st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 4.0f, first_compressed.data(), blocks); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 4.0f, second_compressed.data(), blocks); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, first_compressed.data(), 0.25f, first_restored.data(), blocks, + true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, second_compressed.data(), 0.25f, second_restored.data(), + blocks, true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + EXPECT_EQ(second_compressed, first_compressed); + EXPECT_EQ(second_restored, first_restored); +} + +TEST(FallbackStdparEdgeTest, TailWidthCompressMatchesScalarF32) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < EPB; ++i) { + input[i] = static_cast(static_cast(i) - 10); + } + + std::array scalar_block{}; + std::array stdpar_block{}; + std::array stdpar_blocks{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, scalar_block.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, stdpar_block.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0f, stdpar_blocks.data(), 1); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + for (u32 i = 0; i < BS; ++i) { + EXPECT_EQ(stdpar_block[i], scalar_block[i]) << "block byte " << i; + EXPECT_EQ(stdpar_blocks[i], scalar_block[i]) << "blocks byte " << i; + } +} + +TEST(FallbackStdparEdgeTest, TailWidthCompressMatchesScalarF64) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < EPB; ++i) { + input[i] = static_cast((static_cast(i) - 10) * 2); + } + + std::array scalar_block{}; + std::array stdpar_block{}; + std::array stdpar_blocks{}; + + auto st = pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0, scalar_block.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0, stdpar_block.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_compress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, input.data(), 1.0, stdpar_blocks.data(), 1); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + for (u32 i = 0; i < BS; ++i) { + EXPECT_EQ(stdpar_block[i], scalar_block[i]) << "block byte " << i; + EXPECT_EQ(stdpar_blocks[i], scalar_block[i]) << "blocks byte " << i; + } +} + +TEST(FallbackStdparEdgeTest, TailWidthDecompressMatchesScalarF32) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < EPB; ++i) { + input[i] = static_cast(static_cast(i) - 10); + } + + std::array compressed{}; + std::array scalar_block{}; + std::array stdpar_block{}; + std::array stdpar_blocks{}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0f, compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), 1.0f, scalar_block.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, stdpar_block.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0f, stdpar_blocks.data(), 1, true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + for (u32 i = 0; i < EPB; ++i) { + EXPECT_FLOAT_EQ(stdpar_block[i], scalar_block[i]) << "block element " << i; + EXPECT_FLOAT_EQ(stdpar_blocks[i], scalar_block[i]) << "blocks element " << i; + } +} + +TEST(FallbackStdparEdgeTest, TailWidthDecompressMatchesScalarF64) { + constexpr u32 BS = 64; + constexpr u32 BW = 24; + constexpr u32 EPB = (BS * 8) / BW; + + std::array input{}; + for (u32 i = 0; i < EPB; ++i) { + input[i] = static_cast((static_cast(i) - 10) * 2); + } + + std::array compressed{}; + std::array scalar_block{}; + std::array stdpar_block{}; + std::array stdpar_blocks{}; + + auto st = pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, input.data(), 1.0, compressed.data()); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, BW, BS, compressed.data(), 1.0, scalar_block.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0, stdpar_block.data(), true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + st = pernix_decompress_blocks_f64(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, compressed.data(), 1.0, stdpar_blocks.data(), 1, true); + ASSERT_EQ(st, PERNIX_STATUS_OK); + + for (u32 i = 0; i < EPB; ++i) { + EXPECT_DOUBLE_EQ(stdpar_block[i], scalar_block[i]) << "block element " << i; + EXPECT_DOUBLE_EQ(stdpar_blocks[i], scalar_block[i]) << "blocks element " << i; + } +} +#endif + +// --------------------------------------------------------------------------- +// Error-code contract tests +// --------------------------------------------------------------------------- + +TEST(ErrorCodeTest, UnsupportedBlockSizeReturnsError) { + constexpr u32 BS = 32; + f32 src[32] = {}; + u8 dst[32] = {}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); + + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, 8, BS, dst, 1.0f, src, true); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); + + st = pernix_compress_blocks_f32(PERNIX_BACKEND_FALLBACK, 8, BS, src, 1.0f, dst, 1); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); + + st = pernix_decompress_blocks_f32(PERNIX_BACKEND_FALLBACK, 8, BS, dst, 1.0f, src, 1, true); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); +} + +TEST(ErrorCodeTest, UnsupportedBitWidthReturnsError) { + constexpr u32 BS = 64; + f32 src[256] = {}; + u8 dst[64] = {}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 0, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 25, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); +} + +TEST(ErrorCodeTest, NullPointerReturnsError) { + f32 src[64] = {}; + u8 dst[64] = {}; + + auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, 64, nullptr, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_INVALID_ARGUMENT); + + st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, 8, 64, src, 1.0f, nullptr); + EXPECT_EQ(st, PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(ErrorCodeTest, FallbackAliasMatchesScalar) { + EXPECT_EQ(PERNIX_BACKEND_FALLBACK, PERNIX_BACKEND_FALLBACK_SCALAR); +} + +TEST(ErrorCodeTest, FallbackStdparAvailabilityMatchesBuildConfiguration) { + constexpr u32 BS = 64; + constexpr u8 BW = 8; + + f32 src[(BS * 8U) / BW] = {}; + u8 dst[BS] = {}; + +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_OK); +#else + const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_STDPAR, BW, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +#endif +} + +TEST(ErrorCodeTest, FallbackSimdReturnsUnsupportedImplementationByDefault) { + constexpr u32 BS = 64; + constexpr u8 BW = 8; + + f32 src[(BS * 8U) / BW] = {}; + u8 dst[BS] = {}; + + const auto st = pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK_SIMD, BW, BS, src, 1.0f, dst); + EXPECT_EQ(st, PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +} + +TEST(ErrorCodeTest, ExplicitUnavailableBackendReturnsError) { + constexpr u32 BS = 64; + constexpr u8 BW = 8; + + f32 src[(BS * 8U) / BW] = {}; + u8 dst[BS] = {}; + + pernix_backend backend = PERNIX_BACKEND_FALLBACK; + bool found = false; + +#if defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) + if (!__builtin_cpu_supports("avx512vbmi")) { + backend = PERNIX_BACKEND_X86_AVX512_VBMI; + found = true; + } else if (!(__builtin_cpu_supports("avx2") && __builtin_cpu_supports("bmi2"))) { + backend = PERNIX_BACKEND_X86_BMI2; + found = true; + } else if (!__builtin_cpu_supports("avx2")) { + backend = PERNIX_BACKEND_X86_AVX2; + found = true; + } +#elif defined(__aarch64__) || defined(_M_ARM64) + const bool neon = true; +#if defined(__linux__) && defined(__aarch64__) + const bool sve2 = (getauxval(AT_HWCAP2) & HWCAP2_SVE2) != 0; +#else + const bool sve2 = false; +#endif + if (!sve2) { + backend = PERNIX_BACKEND_ARM64_SVE; + found = true; + } else if (!neon) { + backend = PERNIX_BACKEND_ARM64_NEON; + found = true; + } +#endif + + if (!found) { + GTEST_SKIP() << "No compiled explicit backend is unsupported on this machine"; + } + + const auto st = pernix_compress_block_f32(backend, BW, BS, src, 1.0f, dst); + // A backend can be absent from the build, or compiled but unsupported by this CPU. + EXPECT_TRUE(st == PERNIX_STATUS_UNSUPPORTED_BACKEND || st == PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +} diff --git a/tests/header_only_tests.cpp b/tests/header_only_tests.cpp new file mode 100644 index 0000000..a6e41e8 --- /dev/null +++ b/tests/header_only_tests.cpp @@ -0,0 +1,429 @@ +#include + +#include +#include +#include +#include + +#include + +namespace { +constexpr u8 kBitWidth = 8; +constexpr u32 kBlockSize = PERNIX_TEST_BLOCK_SIZE; +constexpr u32 kBlockElements = (kBlockSize * 8U) / kBitWidth; + +template +std::vector make_input() { + std::vector values(kBlockElements); + for (u32 i = 0; i < kBlockElements; ++i) { + values[static_cast(i)] = static_cast((static_cast(i % 13U) - 6) * 0.25); + } + return values; +} + +template +void expect_round_trip(const pernix::Backend backend) { + auto input = make_input(); + std::vector compressed(kBlockSize); + std::vector restored(kBlockElements, static_cast(0)); + + ASSERT_EQ(pernix::compress_block(backend, kBitWidth, kBlockSize, std::span(input), static_cast(1), + std::span(compressed)), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_block(backend, kBitWidth, kBlockSize, std::span(compressed), static_cast(1), + std::span(restored)), + PERNIX_STATUS_OK); + + for (usize i = 0; i < restored.size(); ++i) { + EXPECT_NEAR(restored[i], input[i], static_cast(1.0)); + } +} + +template +std::vector make_block_pattern(const u8 bit_width, const u32 blocks) { + const u32 elements_per_block = pernix::elements_per_block(bit_width); + std::vector values(static_cast(elements_per_block) * blocks); + + for (usize i = 0; i < values.size(); ++i) { + values[i] = static_cast((static_cast(i % 29U) - 14) * 0.25); + } + + return values; +} + +template +void expect_stdpar_matches_scalar_for_all_bit_widths() { + constexpr u32 blocks = 3; + + for (u8 bit_width = pernix::min_bit_width(); bit_width <= pernix::max_bit_width(); ++bit_width) { + const u32 elements_per_block = pernix::elements_per_block(bit_width); + auto input = make_block_pattern(bit_width, blocks); + + std::vector scalar_block(kBlockSize); + std::vector stdpar_block(kBlockSize); + std::vector scalar_blocks(static_cast(kBlockSize) * blocks); + std::vector stdpar_blocks(static_cast(kBlockSize) * blocks); + std::vector scalar_restored(elements_per_block * blocks); + std::vector stdpar_restored(elements_per_block * blocks); + + ASSERT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, + std::span(input.data(), elements_per_block), static_cast(4), + std::span(scalar_block)), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + ASSERT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, + std::span(input.data(), elements_per_block), static_cast(4), + std::span(stdpar_block)), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + EXPECT_EQ(stdpar_block, scalar_block) << "bit_width=" << static_cast(bit_width); + + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), + static_cast(4), std::span(scalar_blocks), blocks), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), + static_cast(4), std::span(stdpar_blocks), blocks), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + EXPECT_EQ(stdpar_blocks, scalar_blocks) << "bit_width=" << static_cast(bit_width); + + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(scalar_blocks), + static_cast(0.25), std::span(scalar_restored), blocks, true), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(stdpar_blocks), + static_cast(0.25), std::span(stdpar_restored), blocks, true), + PERNIX_STATUS_OK) + << "bit_width=" << static_cast(bit_width); + EXPECT_EQ(stdpar_restored, scalar_restored) << "bit_width=" << static_cast(bit_width); + } +} + +template +void expect_cross_compatibility(const u8 bit_width, const u32 blocks, const FloatT compression_scale, const FloatT decompression_scale, + const bool sign_values = true) { + const u32 elements_per_block = pernix::elements_per_block(bit_width); + auto input = make_block_pattern(bit_width, blocks); + + std::vector scalar_compressed(static_cast(kBlockSize) * blocks); + std::vector stdpar_compressed(static_cast(kBlockSize) * blocks); + std::vector scalar_to_stdpar(static_cast(elements_per_block) * blocks, static_cast(0)); + std::vector stdpar_to_scalar(static_cast(elements_per_block) * blocks, static_cast(0)); + + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), compression_scale, + std::span(scalar_compressed), blocks), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), + compression_scale, std::span(stdpar_compressed), blocks), + PERNIX_STATUS_OK); + + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(scalar_compressed), + decompression_scale, std::span(scalar_to_stdpar), blocks, sign_values), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(stdpar_compressed), + decompression_scale, std::span(stdpar_to_scalar), blocks, sign_values), + PERNIX_STATUS_OK); + + EXPECT_EQ(scalar_to_stdpar, stdpar_to_scalar) << "bit_width=" << static_cast(bit_width); +} + +template +void expect_single_block_partial_group_matches_scalar(const u8 bit_width) { + const u32 elements_per_block = pernix::elements_per_block(bit_width); + auto input = make_block_pattern(bit_width, 1); + + ASSERT_GT(elements_per_block, 8U) << "bit_width=" << static_cast(bit_width); + ASSERT_NE(elements_per_block % 8U, 0U) << "bit_width=" << static_cast(bit_width); + + std::vector scalar_compressed(kBlockSize); + std::vector stdpar_compressed(kBlockSize); + std::vector scalar_restored(elements_per_block, static_cast(0)); + std::vector stdpar_restored(elements_per_block, static_cast(0)); + + ASSERT_EQ(pernix::compress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), + static_cast(4), std::span(scalar_compressed)), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), + static_cast(4), std::span(stdpar_compressed)), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_compressed, scalar_compressed) << "bit_width=" << static_cast(bit_width); + + ASSERT_EQ(pernix::decompress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(scalar_compressed), + static_cast(0.25), std::span(scalar_restored), true), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(stdpar_compressed), + static_cast(0.25), std::span(stdpar_restored), true), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_restored, scalar_restored) << "bit_width=" << static_cast(bit_width); +} +} // namespace + +TEST(HeaderOnlyPernix, FallbackFloatRoundTrip) { + expect_round_trip(pernix::Backend::Fallback); +} + +TEST(HeaderOnlyPernix, PublicHelpersDescribeFixedBlockFormat) { + EXPECT_EQ(pernix::min_bit_width(), 1); + EXPECT_EQ(pernix::max_bit_width(), 24); + EXPECT_TRUE(pernix::is_valid_bit_width(1)); + EXPECT_TRUE(pernix::is_valid_bit_width(24)); + EXPECT_FALSE(pernix::is_valid_bit_width(0)); + EXPECT_FALSE(pernix::is_valid_bit_width(25)); + EXPECT_TRUE(pernix::is_valid_block_size(64)); + EXPECT_FALSE(pernix::is_valid_block_size(96)); + EXPECT_EQ(pernix::compressed_block_size(), 64); + EXPECT_EQ(pernix::elements_per_block(16), 32); + EXPECT_EQ(pernix::elements_per_block(0), 0); + + float scale_f32 = 0.0f; + EXPECT_EQ(pernix::scale_from_bmax(32767.0f, 16, scale_f32), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(scale_f32, 1.0f); + + double scale_f64 = 0.0; + EXPECT_EQ(pernix::scale_from_bmax(0.0, 16, scale_f64), PERNIX_STATUS_OK); + EXPECT_GT(scale_f64, 0.0); +} + +TEST(HeaderOnlyPernix, ScaleHelpersNameCompressionAndDecompressionConventions) { + float decompression_scale = 0.0f; + float compression_scale = 0.0f; + float inverse_scale = 0.0f; + + EXPECT_EQ(pernix::decompression_scale_from_bmax(127.0f, 8, decompression_scale), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(decompression_scale, 1.0f); + EXPECT_EQ(pernix::compression_scale_from_bmax(127.0f, 8, compression_scale), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(compression_scale, 1.0f); + EXPECT_EQ(pernix::inverse_scale(decompression_scale, inverse_scale), PERNIX_STATUS_OK); + EXPECT_FLOAT_EQ(inverse_scale, compression_scale); + + EXPECT_EQ(pernix::inverse_scale(0.0f, inverse_scale), PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::compression_scale_from_bmax(1.0f, 0, compression_scale), PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, StatusStringsArePublic) { + pernix::Status status = PERNIX_STATUS_OK; + EXPECT_EQ(status, PERNIX_STATUS_OK); + EXPECT_STREQ(pernix::status_string(status), "PERNIX_STATUS_OK"); + EXPECT_STREQ(pernix::status_string(PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION), "PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION"); + EXPECT_STREQ(pernix::status_string(static_cast(123)), "PERNIX_STATUS_UNKNOWN"); +} + +TEST(HeaderOnlyPernix, RejectsUndersizedCompressInputSpan) { + std::vector input(kBlockElements - 1U, 0.0f); + std::vector output(kBlockSize); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(output)), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, RejectsUndersizedCompressOutputSpan) { + std::vector input(kBlockElements, 0.0f); + std::vector output(kBlockSize - 1U); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(output)), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, RejectsUndersizedDecompressSpans) { + std::vector short_input(kBlockSize - 1U); + std::vector output(kBlockElements); + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(short_input), 1.0f, + std::span(output)), + PERNIX_STATUS_INVALID_ARGUMENT); + + std::vector input(kBlockSize); + std::vector short_output(kBlockElements - 1U); + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(short_output)), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, RejectsInvalidSpanParametersBeforeDispatch) { + std::vector input(kBlockElements, 0.0f); + std::vector output(kBlockSize); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, 0, kBlockSize, std::span(input), 1.0f, std::span(output)), + PERNIX_STATUS_UNSUPPORTED_BIT_WIDTH); + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, 96, std::span(input), 1.0f, std::span(output)), + PERNIX_STATUS_UNSUPPORTED_BLOCK_SIZE); + EXPECT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(output), 0), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, RejectsInvalidScale) { + std::vector input(kBlockElements, 0.0f); + std::vector compressed(kBlockSize); + std::vector output(input.size()); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), 0.0f, + std::span(compressed)), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::decompress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(compressed), -1.0f, + std::span(output)), + PERNIX_STATUS_INVALID_ARGUMENT); + EXPECT_EQ(pernix::compress_block(pernix::Backend::Fallback, kBitWidth, kBlockSize, std::span(input), + std::numeric_limits::infinity(), std::span(compressed)), + PERNIX_STATUS_INVALID_ARGUMENT); +} + +TEST(HeaderOnlyPernix, FallbackAliasMatchesScalar) { + EXPECT_EQ(static_cast(pernix::Backend::Fallback), static_cast(pernix::Backend::FallbackScalar)); +} + +TEST(HeaderOnlyPernix, FallbackStdparReturnsUnsupportedImplementation) { + auto input = make_input(); + std::vector compressed(kBlockSize); + +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) + EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(compressed)), + PERNIX_STATUS_OK); +#else + EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(compressed)), + PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +#endif +} + +#if defined(PERNIX_BUILD_FALLBACK_STDPAR) +TEST(HeaderOnlyPernix, FallbackStdparMatchesScalarForAllBitWidthsF32) { + expect_stdpar_matches_scalar_for_all_bit_widths(); +} + +TEST(HeaderOnlyPernix, FallbackStdparMatchesScalarForAllBitWidthsF64) { + expect_stdpar_matches_scalar_for_all_bit_widths(); +} + +TEST(HeaderOnlyPernix, FallbackStdparMatchesScalarForEdgeValues) { + constexpr u8 bit_width = 4; + constexpr u32 blocks = 2; + + const u32 elements_per_block = pernix::elements_per_block(bit_width); + std::vector input(static_cast(elements_per_block) * blocks, 0.0f); + input[0] = 0.49f; + input[1] = 0.50f; + input[2] = 1.49f; + input[3] = 1.50f; + input[4] = std::numeric_limits::infinity(); + input[5] = -std::numeric_limits::infinity(); + input[6] = std::numeric_limits::quiet_NaN(); + input[7] = 99.0f; + input[8] = -99.0f; + + std::vector scalar_compressed(static_cast(kBlockSize) * blocks); + std::vector stdpar_compressed(static_cast(kBlockSize) * blocks); + std::vector scalar_signed(input.size()); + std::vector stdpar_signed(input.size()); + + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(input), 1.0f, + std::span(scalar_compressed), blocks), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 1.0f, + std::span(stdpar_compressed), blocks), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_compressed, scalar_compressed); + + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(scalar_compressed), 1.0f, + std::span(scalar_signed), blocks, true), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(stdpar_compressed), + 1.0f, std::span(stdpar_signed), blocks, true), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_signed, scalar_signed); +} + +TEST(HeaderOnlyPernix, FallbackStdparSingleBlockPartialGroupsMatchScalar) { + for (const u8 bit_width : std::array{3, 5, 6, 7, 10, 11, 12, 24}) { + expect_single_block_partial_group_matches_scalar(bit_width); + } +} + +TEST(HeaderOnlyPernix, FallbackStdparCrossCompatibilityMatchesScalarForSignedValues) { + for (const u8 bit_width : std::array{1, 3, 5, 8, 9, 12, 17, 24}) { + expect_cross_compatibility(bit_width, 2, 4.0f, 0.25f, true); + } +} + +TEST(HeaderOnlyPernix, FallbackStdparCrossCompatibilityMatchesScalarForUnsignedDecode) { + expect_cross_compatibility(4, 2, 1.0f, 1.0f, false); +} + +TEST(HeaderOnlyPernix, FallbackStdparSignValuesFalseMatchesScalar) { + constexpr u8 bit_width = 4; + const u32 elements_per_block = pernix::elements_per_block(bit_width); + + std::vector compressed(kBlockSize, 0); + compressed[0] = 0x0F; + + std::vector scalar_output(elements_per_block, 0.0f); + std::vector stdpar_output(elements_per_block, 0.0f); + + ASSERT_EQ(pernix::decompress_block(pernix::Backend::Fallback, bit_width, kBlockSize, std::span(compressed), 1.0f, + std::span(scalar_output), false), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(compressed), 1.0f, + std::span(stdpar_output), false), + PERNIX_STATUS_OK); + EXPECT_EQ(stdpar_output, scalar_output); +} + +TEST(HeaderOnlyPernix, FallbackStdparRepeatedExecutionIsDeterministic) { + constexpr u8 bit_width = 12; + constexpr u32 blocks = 3; + + auto input = make_block_pattern(bit_width, blocks); + std::vector first_compressed(static_cast(kBlockSize) * blocks); + std::vector second_compressed(static_cast(kBlockSize) * blocks); + std::vector first_restored(input.size(), 0.0f); + std::vector second_restored(input.size(), 0.0f); + + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 4.0f, + std::span(first_compressed), blocks), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 4.0f, + std::span(second_compressed), blocks), + PERNIX_STATUS_OK); + EXPECT_EQ(second_compressed, first_compressed); + + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(first_compressed), + 0.25f, std::span(first_restored), blocks, true), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::decompress_blocks(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(second_compressed), + 0.25f, std::span(second_restored), blocks, true), + PERNIX_STATUS_OK); + EXPECT_EQ(second_restored, first_restored); +} + +TEST(HeaderOnlyPernix, FallbackStdparRepeatedSingleBlockCompressionIsDeterministic) { + constexpr u8 bit_width = 17; + + auto input = make_block_pattern(bit_width, 1); + std::vector first_compressed(kBlockSize); + std::vector second_compressed(kBlockSize); + + ASSERT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 2.0, + std::span(first_compressed)), + PERNIX_STATUS_OK); + ASSERT_EQ(pernix::compress_block(pernix::Backend::FallbackStdpar, bit_width, kBlockSize, std::span(input), 2.0, + std::span(second_compressed)), + PERNIX_STATUS_OK); + EXPECT_EQ(second_compressed, first_compressed); +} +#endif + +TEST(HeaderOnlyPernix, FallbackSimdReturnsUnsupportedImplementation) { + auto input = make_input(); + std::vector compressed(kBlockSize); + + EXPECT_EQ(pernix::compress_block(pernix::Backend::FallbackSimd, kBitWidth, kBlockSize, std::span(input), 1.0f, + std::span(compressed)), + PERNIX_STATUS_UNSUPPORTED_IMPLEMENTATION); +} + +TEST(HeaderOnlyPernix, AutoDoubleRoundTrip) { + expect_round_trip(pernix::Backend::Auto); +} diff --git a/tests/include/testset.h b/tests/include/testset.h index 6535957..87054af 100644 --- a/tests/include/testset.h +++ b/tests/include/testset.h @@ -1,42 +1,34 @@ #ifndef PERNIX_TESTSET_H #define PERNIX_TESTSET_H -#include <../../include/pernix/pernix.h> -#include +#include #include -#include -#include #include +#include +#include #include #include #include #include #include +#include + #ifndef PERNIX_TEST_BLOCK_SIZE #define PERNIX_TEST_BLOCK_SIZE 64 #endif -static_assert(PERNIX_TEST_BLOCK_SIZE % 32 == 0, - "PERNIX_TEST_BLOCK_SIZE must be dividable by 32 bytes"); - -/** - * A test set for compression and decompression tests. - * It generates random float data, compresses it, and verifies the decompression using the fallback implementation. - * - * @tparam BIT_WIDTH The bit width used for compression (1 to 24). - * @tparam SIGN_VALUES Indicates whether the values are signed or unsigned. - */ -template +static_assert(PERNIX_TEST_BLOCK_SIZE % 32 == 0, "PERNIX_TEST_BLOCK_SIZE must be dividable by 32 bytes"); + +template requires(BIT_WIDTH >= 1 && BIT_WIDTH <= 24) && std::is_floating_point_v class TestSet { - // using ValueType = std::conditional_t; - using ValueType = uint8_t; + using ValueType = u8; using SeedType = std::mt19937::result_type; alignas(64) std::vector > compressedData; - alignas(64) std::vector > decompressedData; + alignas(64) std::vector > sourceData; alignas(64) std::vector scalesData; SeedType seed; @@ -44,25 +36,24 @@ class TestSet { std::uniform_real_distribution dis{}; public: - static constexpr uint32_t elementsPerBlock = (BLOCK_SIZE * 8) / BIT_WIDTH; - static constexpr SeedType defaultSeed = 0x5eed1234u; + static constexpr u32 elementsPerBlock = (BLOCK_SIZE * 8) / BIT_WIDTH; + static constexpr SeedType defaultSeed = 0x5eed1234u; static constexpr T quantization_levels = SIGN_VALUES ? static_cast(BIT_WIDTH == 1 ? 1u : ((1u << (BIT_WIDTH - 1u)) - 1u)) : static_cast((1u << BIT_WIDTH) - 1u); - uint32_t numberOfBlocks; + u32 numberOfBlocks; - [[nodiscard]] constexpr uint32_t totalElements() const { return numberOfBlocks * elementsPerBlock; } + [[nodiscard]] constexpr u32 totalElements() const { return numberOfBlocks * elementsPerBlock; } - [[nodiscard]] T blockTolerance(const uint32_t block) const { - // Half-step quantization bound + tiny FP slack for rounding edge cases. + [[nodiscard]] T blockTolerance(const u32 block) const { return (std::abs(scalesData[block]) * static_cast(0.5)) + (std::numeric_limits::epsilon() * static_cast(16)); } - explicit TestSet(const uint32_t number_of_blocks, const SeedType initial_seed = testSeed()) + explicit TestSet(const u32 number_of_blocks, const SeedType initial_seed = testSeed()) : seed(initial_seed), gen(seed), numberOfBlocks(number_of_blocks) { compressedData.resize(numberOfBlocks); - decompressedData.resize(number_of_blocks); + sourceData.resize(number_of_blocks); scalesData.resize(numberOfBlocks); generateData(); @@ -72,7 +63,7 @@ class TestSet { [[nodiscard]] const std::vector >& getCompressedData() const { return compressedData; } - [[nodiscard]] const std::vector >& getDecompressedData() const { return decompressedData; } + [[nodiscard]] const std::vector >& getDecompressedData() const { return sourceData; } [[nodiscard]] SeedType getSeed() const { return seed; } @@ -88,56 +79,57 @@ class TestSet { } private: - // Generate deterministic source data and its fallback-compressed reference. void generateData() { - for (uint32_t i = 0; i < numberOfBlocks; i++) { + for (u32 i = 0; i < numberOfBlocks; i++) { compressedData[i].resize(BLOCK_SIZE); - decompressedData[i].resize(elementsPerBlock); + sourceData[i].resize(elementsPerBlock); - for (uint32_t j = 0; j < elementsPerBlock; j++) { - decompressedData[i][j] = dis(gen); + for (u32 j = 0; j < elementsPerBlock; j++) { + sourceData[i][j] = dis(gen); } - const T b_max = *std::ranges::max_element(decompressedData[i]); - const T b_min = *std::ranges::min_element(decompressedData[i]); + const T b_max = *std::ranges::max_element(sourceData[i]); + const T b_min = *std::ranges::min_element(sourceData[i]); const T b_abs = std::max(std::abs(b_max), std::abs(b_min)); - scalesData[i] = (b_abs > static_cast(0) && quantization_levels > static_cast(0)) - ? (b_abs / quantization_levels) - : std::numeric_limits::epsilon(); - - // Compress the data using the fallback implementation - pernix::compress_block_fallback(decompressedData[i].data(), 1 / scalesData[i], - reinterpret_cast(compressedData[i].data())); + scalesData[i] = (b_abs > static_cast(0) && quantization_levels > static_cast(0)) ? (b_abs / quantization_levels) + : std::numeric_limits::epsilon(); + + if constexpr (std::is_same_v) { + pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, sourceData[i].data(), 1.0f / scalesData[i], + compressedData[i].data()); + } else { + pernix_compress_block_f64(PERNIX_BACKEND_FALLBACK, BIT_WIDTH, BLOCK_SIZE, sourceData[i].data(), 1.0 / scalesData[i], + compressedData[i].data()); + } } } }; -template +template struct BitWidthBlockSize { - static constexpr uint8_t bit_width = BIT_WIDTH; - static constexpr uint32_t block_size = BLOCK_SIZE; + static constexpr u8 bit_width = BIT_WIDTH; + static constexpr u32 block_size = BLOCK_SIZE; }; using testing::Types; -using BitWidthBlockSizeTypes = -Types, BitWidthBlockSize<2, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<3, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<4, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<5, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<6, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<7, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<8, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<9, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<10, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<11, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<12, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<13, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<14, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<15, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<16, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<17, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<18, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<19, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<20, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<21, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<22, PERNIX_TEST_BLOCK_SIZE>, - BitWidthBlockSize<23, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<24, PERNIX_TEST_BLOCK_SIZE> >; +using BitWidthBlockSizeTypes = Types, BitWidthBlockSize<2, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<3, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<4, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<5, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<6, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<7, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<8, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<9, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<10, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<11, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<12, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<13, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<14, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<15, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<16, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<17, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<18, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<19, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<20, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<21, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<22, PERNIX_TEST_BLOCK_SIZE>, + BitWidthBlockSize<23, PERNIX_TEST_BLOCK_SIZE>, BitWidthBlockSize<24, PERNIX_TEST_BLOCK_SIZE> >; struct BitWidthBlockSizeName { template static std::string GetName(int) { std::ostringstream name; - name << "BitWidth" << static_cast(TestConfigT::bit_width) << "BlockSize" << TestConfigT::block_size; + name << "BitWidth" << static_cast(TestConfigT::bit_width) << "BlockSize" << TestConfigT::block_size; return name.str(); } }; @@ -145,82 +137,78 @@ struct BitWidthBlockSizeName { template class CompressionTest : public ::testing::Test { public: - static constexpr uint8_t BitWidth = TestConfigT::bit_width; - static constexpr uint32_t BlockSize = TestConfigT::block_size; + static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u32 BlockSize = TestConfigT::block_size; - TestSet testSet; + TestSet testSet; - CompressionTest() : testSet(1u << 10) { - } + CompressionTest() : testSet(1u << 10) {} }; template class DecompressionTest : public ::testing::Test { public: - static constexpr uint8_t BitWidth = TestConfigT::bit_width; - static constexpr uint32_t BlockSize = TestConfigT::block_size; + static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u32 BlockSize = TestConfigT::block_size; - TestSet testSet; + TestSet testSet; - DecompressionTest() : testSet(1u << 10) { - } + DecompressionTest() : testSet(1u << 10) {} }; template class CompressionTest64 : public ::testing::Test { public: - static constexpr uint8_t BitWidth = TestConfigT::bit_width; - static constexpr uint32_t BlockSize = TestConfigT::block_size; + static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u32 BlockSize = TestConfigT::block_size; - TestSet testSet; + TestSet testSet; - CompressionTest64() : testSet(1u << 10) { - } + CompressionTest64() : testSet(1u << 10) {} }; template class DecompressionTest64 : public ::testing::Test { public: - static constexpr uint8_t BitWidth = TestConfigT::bit_width; - static constexpr uint32_t BlockSize = TestConfigT::block_size; + static constexpr u8 BitWidth = TestConfigT::bit_width; + static constexpr u32 BlockSize = TestConfigT::block_size; - TestSet testSet; + TestSet testSet; - DecompressionTest64() : testSet(1u << 10) { - } + DecompressionTest64() : testSet(1u << 10) {} }; template -[[nodiscard]] std::string testContext(const FixtureT& fixture, const uint32_t block) { +[[nodiscard]] std::string testContext(const FixtureT& fixture, const u32 block) { std::ostringstream message; - message << "bit_width=" << static_cast(FixtureT::BitWidth) << ", block_size=" << FixtureT::BlockSize - << ", block=" << block << ", scale=" << fixture.testSet.getScales()[block] - << ", tolerance=" << fixture.testSet.blockTolerance(block) << ", seed=" << fixture.testSet.getSeed(); + message << "bit_width=" << static_cast(FixtureT::BitWidth) << ", block_size=" << FixtureT::BlockSize << ", block=" << block + << ", scale=" << fixture.testSet.getScales()[block] << ", tolerance=" << fixture.testSet.blockTolerance(block) + << ", seed=" << fixture.testSet.getSeed(); return message.str(); } template -void expectCompressedBlockEqualsReference(const FixtureT& fixture, const std::vector& actual, const uint32_t block) { +void expectCompressedBlockEqualsReference(const FixtureT& fixture, const std::vector& actual, const u32 block) { SCOPED_TRACE(testContext(fixture, block)); const auto& expected = fixture.testSet.getCompressedData()[block]; ASSERT_EQ(actual.size(), expected.size()) << "Compressed block byte count differs from reference"; - for (uint32_t byte = 0; byte < actual.size(); byte++) { + for (u32 byte = 0; byte < actual.size(); byte++) { ASSERT_EQ(actual[byte], expected[byte]) - << "Compressed byte mismatch at byte=" << byte << ", actual=" << static_cast(actual[byte]) - << ", expected=" << static_cast(expected[byte]); + << "Compressed byte mismatch at byte=" << byte << ", actual=" << static_cast(actual[byte]) + << ", expected=" << static_cast(expected[byte]); } } template -void expectDecompressedBlockNearSource(const FixtureT& fixture, const std::vector& actual, const uint32_t block) { +void expectDecompressedBlockNearSource(const FixtureT& fixture, const std::vector& actual, const u32 block) { SCOPED_TRACE(testContext(fixture, block)); const auto& expected = fixture.testSet.getDecompressedData()[block]; ASSERT_EQ(actual.size(), expected.size()) << "Decompressed block element count differs from source"; - for (uint32_t element = 0; element < actual.size(); element++) { + for (u32 element = 0; element < actual.size(); element++) { ASSERT_NEAR(actual[element], expected[element], fixture.testSet.blockTolerance(block)) << "Decompressed element mismatch at element=" << element << ", actual=" << actual[element] << ", expected=" << expected[element] << ", absolute_error=" << std::abs(actual[element] - expected[element]); @@ -228,8 +216,11 @@ void expectDecompressedBlockNearSource(const FixtureT& fixture, const std::vecto } TYPED_TEST_SUITE(CompressionTest, BitWidthBlockSizeTypes, BitWidthBlockSizeName); + TYPED_TEST_SUITE(DecompressionTest, BitWidthBlockSizeTypes, BitWidthBlockSizeName); + TYPED_TEST_SUITE(CompressionTest64, BitWidthBlockSizeTypes, BitWidthBlockSizeName); + TYPED_TEST_SUITE(DecompressionTest64, BitWidthBlockSizeTypes, BitWidthBlockSizeName); #endif // PERNIX_TESTSET_H diff --git a/tests/install_consumer/CMakeLists.txt b/tests/install_consumer/CMakeLists.txt new file mode 100644 index 0000000..83ed283 --- /dev/null +++ b/tests/install_consumer/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.24) +project(pernix_install_consumer LANGUAGES C CXX) + +find_package(pernix CONFIG REQUIRED) + +add_executable(pernix_consumer_cpp main.cpp) +target_link_libraries(pernix_consumer_cpp PRIVATE pernix::pernix) + +add_executable(pernix_consumer_c main.c) +target_link_libraries(pernix_consumer_c PRIVATE pernix::pernix) + +enable_testing() +add_test(NAME pernix_consumer_cpp COMMAND pernix_consumer_cpp) +add_test(NAME pernix_consumer_c COMMAND pernix_consumer_c) diff --git a/tests/install_consumer/main.c b/tests/install_consumer/main.c new file mode 100644 index 0000000..b6a521a --- /dev/null +++ b/tests/install_consumer/main.c @@ -0,0 +1,32 @@ +#include + +int main(void) { + enum { + bit_width = 8, + block_size = 64, + elements = (block_size * 8) / bit_width + }; + + float input[elements]; + float restored[elements]; + u8 compressed[block_size]; + + for (int i = 0; i < elements; ++i) { + input[i] = (float)((i % 17) - 8) * 0.125f; + } + + if (pernix_compress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, input, 8.0f, compressed) != PERNIX_STATUS_OK) { + return 1; + } + if (pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, bit_width, block_size, compressed, 0.125f, restored, true) != + PERNIX_STATUS_OK) { + return 2; + } + + for (int i = 0; i < elements; ++i) { + if (input[i] != restored[i]) { + return 3; + } + } + return 0; +} diff --git a/tests/install_consumer/main.cpp b/tests/install_consumer/main.cpp new file mode 100644 index 0000000..0094d42 --- /dev/null +++ b/tests/install_consumer/main.cpp @@ -0,0 +1,33 @@ +#include + +#include +#include +#include + +int main() { + constexpr u8 bit_width = 8; + constexpr u32 block_size = 64; + constexpr usize elements = (block_size * 8U) / bit_width; + + std::array input{}; + for (usize i = 0; i < input.size(); ++i) { + input[i] = static_cast(static_cast(i % 17U) - 8) * 0.125f; + } + + std::array compressed{}; + std::array restored{}; + + if (pernix::compress_block(pernix::Backend::Fallback, bit_width, block_size, input, 8.0f, compressed) != PERNIX_STATUS_OK) { + return 1; + } + if (pernix::decompress_block(pernix::Backend::Fallback, bit_width, block_size, compressed, 0.125f, restored) != PERNIX_STATUS_OK) { + return 2; + } + + for (usize i = 0; i < input.size(); ++i) { + if (std::abs(input[i] - restored[i]) > 0.0f) { + return 3; + } + } + return 0; +} diff --git a/tests/install_consumer/run.cmake b/tests/install_consumer/run.cmake new file mode 100644 index 0000000..763a8e5 --- /dev/null +++ b/tests/install_consumer/run.cmake @@ -0,0 +1,69 @@ +if (NOT DEFINED PERNIX_BUILD_DIR) + message(FATAL_ERROR "PERNIX_BUILD_DIR is required") +endif () +if (NOT DEFINED PERNIX_CONSUMER_SOURCE_DIR) + message(FATAL_ERROR "PERNIX_CONSUMER_SOURCE_DIR is required") +endif () +if (NOT DEFINED PERNIX_CONSUMER_BUILD_DIR) + message(FATAL_ERROR "PERNIX_CONSUMER_BUILD_DIR is required") +endif () +if (NOT DEFINED PERNIX_INSTALL_PREFIX) + message(FATAL_ERROR "PERNIX_INSTALL_PREFIX is required") +endif () + +set(PERNIX_CONFIG_ARG) +if (DEFINED PERNIX_TEST_CONFIG AND NOT PERNIX_TEST_CONFIG STREQUAL "") + list(APPEND PERNIX_CONFIG_ARG --config "${PERNIX_TEST_CONFIG}") +endif () + +file(REMOVE_RECURSE "${PERNIX_INSTALL_PREFIX}" "${PERNIX_CONSUMER_BUILD_DIR}") + +execute_process( + COMMAND "${CMAKE_COMMAND}" --install "${PERNIX_BUILD_DIR}" --prefix "${PERNIX_INSTALL_PREFIX}" ${PERNIX_CONFIG_ARG} + RESULT_VARIABLE PERNIX_INSTALL_RESULT +) +if (NOT PERNIX_INSTALL_RESULT EQUAL 0) + message(FATAL_ERROR "Installing pernix for consumer test failed") +endif () + +set(PERNIX_COMPILER_ARGS) +if (DEFINED PERNIX_C_COMPILER AND NOT PERNIX_C_COMPILER STREQUAL "") + list(APPEND PERNIX_COMPILER_ARGS "-DCMAKE_C_COMPILER=${PERNIX_C_COMPILER}") +endif () +if (DEFINED PERNIX_CXX_COMPILER AND NOT PERNIX_CXX_COMPILER STREQUAL "") + list(APPEND PERNIX_COMPILER_ARGS "-DCMAKE_CXX_COMPILER=${PERNIX_CXX_COMPILER}") +endif () + +cmake_host_system_information(RESULT PERNIX_PARALLEL_JOBS QUERY NUMBER_OF_LOGICAL_CORES) +if (NOT PERNIX_PARALLEL_JOBS OR PERNIX_PARALLEL_JOBS LESS 1) + set(PERNIX_PARALLEL_JOBS 1) +endif () + +execute_process( + COMMAND "${CMAKE_COMMAND}" + -S "${PERNIX_CONSUMER_SOURCE_DIR}" + -B "${PERNIX_CONSUMER_BUILD_DIR}" + "-DCMAKE_PREFIX_PATH=${PERNIX_INSTALL_PREFIX}" + "-DCMAKE_BUILD_TYPE=Release" + ${PERNIX_COMPILER_ARGS} + RESULT_VARIABLE PERNIX_CONFIGURE_RESULT +) +if (NOT PERNIX_CONFIGURE_RESULT EQUAL 0) + message(FATAL_ERROR "Configuring installed pernix consumer failed") +endif () + +execute_process( + COMMAND "${CMAKE_COMMAND}" --build "${PERNIX_CONSUMER_BUILD_DIR}" --config Release --parallel "${PERNIX_PARALLEL_JOBS}" + RESULT_VARIABLE PERNIX_BUILD_RESULT +) +if (NOT PERNIX_BUILD_RESULT EQUAL 0) + message(FATAL_ERROR "Building installed pernix consumer failed") +endif () + +execute_process( + COMMAND "${CMAKE_CTEST_COMMAND}" --test-dir "${PERNIX_CONSUMER_BUILD_DIR}" --output-on-failure + RESULT_VARIABLE PERNIX_TEST_RESULT +) +if (NOT PERNIX_TEST_RESULT EQUAL 0) + message(FATAL_ERROR "Running installed pernix consumer tests failed") +endif () diff --git a/tests/pack_tables_tests.cpp b/tests/pack_tables_tests.cpp new file mode 100644 index 0000000..c0ddabe --- /dev/null +++ b/tests/pack_tables_tests.cpp @@ -0,0 +1,154 @@ +#include +#include + +#include +#include +#include + +#include + +namespace { +static_assert(pernix::detail::low_bit_mask<1>() == 0x00000001U); +static_assert(pernix::detail::low_bit_mask<7>() == 0x0000007fU); +static_assert(pernix::detail::low_bit_mask<8>() == 0x000000ffU); +static_assert(pernix::detail::low_bit_mask<9>() == 0x000001ffU); +static_assert(pernix::detail::low_bit_mask<15>() == 0x00007fffU); +static_assert(pernix::detail::low_bit_mask<16>() == 0x0000ffffU); +static_assert(pernix::detail::low_bit_mask<17>() == 0x0001ffffU); +static_assert(pernix::detail::low_bit_mask<24>() == 0x00ffffffU); + +template +constexpr bool validate_pack_table() { + using table = pernix::detail::pack_table; + using unsigned_type = std::make_unsigned_t; + + constexpr usize elements = VectorBytes / sizeof(LaneType); + constexpr usize lane_bits = sizeof(LaneType) * 8; + constexpr unsigned_type value_mask = (unsigned_type{1} << BitWidth) - 1; + constexpr unsigned_type lane_mask = static_cast(~unsigned_type{0}); + + std::array, table::contributor_count> reference_primary_permutes{}; + std::array, table::contributor_count> reference_primary_shifts{}; + std::array reference_primary_masks{}; + std::array reference_spill_permute{}; + std::array reference_spill_shift{}; + reference_spill_permute.fill(static_cast(-1)); + reference_spill_shift.fill(static_cast(lane_bits)); + for (usize contributor = 0; contributor < table::contributor_count; ++contributor) { + reference_primary_permutes[contributor].fill(static_cast(-1)); + reference_primary_shifts[contributor].fill(static_cast(lane_bits)); + } + + u64 reference_spill_mask = 0; + for (usize output_lane = 0; output_lane < elements; ++output_lane) { + const usize output_start = output_lane * lane_bits; + usize contributor = 0; + usize previous_input = elements; + + // Derive the plan independently from each packed output bit rather than interval arithmetic. + for (usize output_bit = 0; output_bit < lane_bits; ++output_bit) { + const usize packed_bit = output_start + output_bit; + const usize input_lane = packed_bit / BitWidth; + if (input_lane >= elements || input_lane == previous_input) { + continue; + } + previous_input = input_lane; + const auto shift = static_cast(input_lane * BitWidth) - static_cast(output_start); + if (shift < 0) { + reference_spill_permute[output_lane] = static_cast(input_lane); + reference_spill_shift[output_lane] = static_cast(-shift); + reference_spill_mask |= u64{1} << output_lane; + } else { + if (contributor >= table::contributor_count) { + return false; + } + reference_primary_permutes[contributor][output_lane] = static_cast(input_lane); + reference_primary_shifts[contributor][output_lane] = static_cast(shift); + reference_primary_masks[contributor] |= u64{1} << output_lane; + ++contributor; + } + } + } + + if (table::primary_permutes != reference_primary_permutes || table::primary_shifts != reference_primary_shifts || + table::primary_masks != reference_primary_masks || table::spill_permute != reference_spill_permute || + table::spill_shift != reference_spill_shift || table::spill_mask != reference_spill_mask) { + return false; + } + + std::array input{}; + for (usize lane = 0; lane < elements; ++lane) { + input[lane] = static_cast(((lane * 0x9e37U) + 0x5a5aU) & value_mask); + } + + for (usize output_lane = 0; output_lane < elements; ++output_lane) { + unsigned_type generated = 0; + const auto add_left = [&](const LaneType index, const LaneType shift) { + if (index >= 0) { + generated |= static_cast((input[static_cast(index)] << shift) & lane_mask); + } + }; + for (usize slot = 0; slot < table::contributor_count; ++slot) { + add_left(table::primary_permutes[slot][output_lane], table::primary_shifts[slot][output_lane]); + } + if (table::spill_permute[output_lane] >= 0) { + generated |= + static_cast(input[static_cast(table::spill_permute[output_lane])] >> table::spill_shift[output_lane]); + } + + unsigned_type expected = 0; + for (usize bit = 0; bit < lane_bits; ++bit) { + const usize packed_bit = (output_lane * lane_bits) + bit; + const usize input_lane = packed_bit / BitWidth; + const usize input_bit = packed_bit % BitWidth; + if (input_lane < elements) { + expected |= static_cast(((input[input_lane] >> input_bit) & 1U) << bit); + } + } + if (generated != expected) { + return false; + } + + const auto mask_bit = u64{1} << output_lane; + for (usize slot = 0; slot < table::contributor_count; ++slot) { + if (((table::primary_masks[slot] & mask_bit) != 0) != (table::primary_permutes[slot][output_lane] >= 0)) { + return false; + } + } + if (((table::spill_mask & mask_bit) != 0) != (table::spill_permute[output_lane] >= 0)) { + return false; + } + } + + return true; +} + +template +constexpr bool validate_8_bit_tables(std::index_sequence) { + return (validate_pack_table(Indices + 1), VectorBytes>() && ...); +} + +template +constexpr bool validate_16_bit_tables(std::index_sequence) { + return (validate_pack_table(Indices + 9), VectorBytes>() && ...); +} + +template +constexpr bool validate_32_bit_tables(std::index_sequence) { + return (validate_pack_table(Indices + 17), VectorBytes>() && ...); +} + +static_assert(validate_8_bit_tables<16>(std::make_index_sequence<8>{})); +static_assert(validate_8_bit_tables<32>(std::make_index_sequence<8>{})); +static_assert(validate_8_bit_tables<64>(std::make_index_sequence<8>{})); +static_assert(validate_16_bit_tables<16>(std::make_index_sequence<8>{})); +static_assert(validate_16_bit_tables<32>(std::make_index_sequence<8>{})); +static_assert(validate_16_bit_tables<64>(std::make_index_sequence<8>{})); +static_assert(validate_32_bit_tables<16>(std::make_index_sequence<8>{})); +static_assert(validate_32_bit_tables<32>(std::make_index_sequence<8>{})); +static_assert(validate_32_bit_tables<64>(std::make_index_sequence<8>{})); +} // namespace + +TEST(PackTablesTest, GeneratesAllAvx512LanePlansAtCompileTime) { + SUCCEED(); +} diff --git a/tests/simd_tests.cpp b/tests/simd_tests.cpp new file mode 100644 index 0000000..0e01de5 --- /dev/null +++ b/tests/simd_tests.cpp @@ -0,0 +1,187 @@ +#include +#include +#include + +// --------------------------------------------------------------------------- +// SIMD compress: compress via backend, decompress via fallback, compare source +// --------------------------------------------------------------------------- + +template +void testBackendCompressBlock(FixtureT& fixture, pernix_backend backend) { + using T = std::remove_cvref_t; + + { + std::vector probe(FixtureT::BlockSize); + pernix_status st; + if constexpr (std::is_same_v) { + st = + pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[0].data(), + 1.0f / fixture.testSet.getScales()[0], probe.data()); + } else { + st = + pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[0].data(), + 1.0 / fixture.testSet.getScales()[0], probe.data()); + } + if (st != PERNIX_STATUS_OK) { + GTEST_SKIP(); + return; + } + } + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; b++) { + std::vector compressed(FixtureT::BlockSize); + + pernix_status st; + if constexpr (std::is_same_v) { + st = + pernix_compress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[b].data(), + 1.0f / fixture.testSet.getScales()[b], compressed.data()); + } else { + st = + pernix_compress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getDecompressedData()[b].data(), + 1.0 / fixture.testSet.getScales()[b], compressed.data()); + } + ASSERT_EQ(st, PERNIX_STATUS_OK); + + std::vector restored(fixture.testSet.elementsPerBlock); + if constexpr (std::is_same_v) { + st = pernix_decompress_block_f32(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, compressed.data(), + fixture.testSet.getScales()[b], restored.data(), true); + } else { + st = pernix_decompress_block_f64(PERNIX_BACKEND_FALLBACK, FixtureT::BitWidth, FixtureT::BlockSize, compressed.data(), + fixture.testSet.getScales()[b], restored.data(), true); + } + ASSERT_EQ(st, PERNIX_STATUS_OK); + + expectDecompressedBlockNearSource(fixture, restored, b); + } +} + +// --------------------------------------------------------------------------- +// SIMD decompress: decompress fallback-compressed data via backend, compare source +// --------------------------------------------------------------------------- + +template +void testBackendDecompressBlock(FixtureT& fixture, pernix_backend backend) { + using T = std::remove_cvref_t; + + { + std::vector probe(fixture.testSet.elementsPerBlock); + pernix_status st; + if constexpr (std::is_same_v) { + st = + pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[0].data(), + fixture.testSet.getScales()[0], probe.data(), true); + } else { + st = + pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[0].data(), + fixture.testSet.getScales()[0], probe.data(), true); + } + if (st != PERNIX_STATUS_OK) { + GTEST_SKIP(); + return; + } + } + + for (u32 b = 0; b < fixture.testSet.numberOfBlocks; b++) { + std::vector decompressed(fixture.testSet.elementsPerBlock); + + pernix_status st; + if constexpr (std::is_same_v) { + st = + pernix_decompress_block_f32(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed.data(), true); + } else { + st = + pernix_decompress_block_f64(backend, FixtureT::BitWidth, FixtureT::BlockSize, fixture.testSet.getCompressedData()[b].data(), + fixture.testSet.getScales()[b], decompressed.data(), true); + } + ASSERT_EQ(st, PERNIX_STATUS_OK); + + expectDecompressedBlockNearSource(fixture, decompressed, b); + } +} + +// --------------------------------------------------------------------------- +// x86: AVX2 +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, AVX2CompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_AVX2); +} + +TYPED_TEST(DecompressionTest, AVX2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_AVX2); +} + +TYPED_TEST(CompressionTest64, AVX2CompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_AVX2); +} + +TYPED_TEST(DecompressionTest64, AVX2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_AVX2); +} + +// --------------------------------------------------------------------------- +// x86: BMI2 +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, BMI2CompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_BMI2); +} + +TYPED_TEST(DecompressionTest, BMI2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_BMI2); +} + +TYPED_TEST(CompressionTest64, BMI2CompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_BMI2); +} + +TYPED_TEST(DecompressionTest64, BMI2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_BMI2); +} + +// --------------------------------------------------------------------------- +// x86: AVX512-VBMI +// --------------------------------------------------------------------------- + +TYPED_TEST(CompressionTest, AVX512VBMICompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_AVX512_VBMI); +} + +TYPED_TEST(DecompressionTest, AVX512VBMIDecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_AVX512_VBMI); +} + +TYPED_TEST(CompressionTest64, AVX512VBMICompressBlock) { + testBackendCompressBlock(*this, PERNIX_BACKEND_X86_AVX512_VBMI); +} + +TYPED_TEST(DecompressionTest64, AVX512VBMIDecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_X86_AVX512_VBMI); +} + +// --------------------------------------------------------------------------- +// ARM64: NEON (decompress only — no compress implementation) +// --------------------------------------------------------------------------- + +TYPED_TEST(DecompressionTest, NeonDecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_ARM64_NEON); +} + +TYPED_TEST(DecompressionTest64, NeonDecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_ARM64_NEON); +} + +// --------------------------------------------------------------------------- +// ARM64: SVE2 (decompress only — no compress implementation) +// --------------------------------------------------------------------------- + +TYPED_TEST(DecompressionTest, SVE2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_ARM64_SVE); +} + +TYPED_TEST(DecompressionTest64, SVE2DecompressBlock) { + testBackendDecompressBlock(*this, PERNIX_BACKEND_ARM64_SVE); +} diff --git a/tests/unpack_tables_tests.cpp b/tests/unpack_tables_tests.cpp new file mode 100644 index 0000000..b7f3db6 --- /dev/null +++ b/tests/unpack_tables_tests.cpp @@ -0,0 +1,93 @@ +#include + +#include +#include + +#include + +namespace { +template +using shift_type_for = std::conditional_t<(BitWidth <= 8), i8, std::conditional_t<(BitWidth <= 16), i16, i32>>; + +template +constexpr bool validate_unpack_table() { + using table = pernix::detail::unpack_table; + + constexpr usize lane_bytes = sizeof(ShiftType); + constexpr usize lanes = VectorBytes / lane_bytes; + bool const expected_has_spill = false; + + if (table::primary_permute.size() != VectorBytes || table::right_shift.size() != lanes || + table::right_shift_magnitude.size() != lanes) { + return false; + } + + for (usize entry = 0; entry < lanes; ++entry) { + const usize bit_start = entry * BitWidth; + const usize first_byte = bit_start / 8; + const usize bit_offset = bit_start % 8; + const usize base = entry * lane_bytes; + + for (usize byte = 0; byte < lane_bytes; ++byte) { + if (table::primary_permute[base + byte] != first_byte + byte) { + return false; + } + } + if (table::right_shift[entry] != -static_cast(bit_offset)) { + return false; + } + if (table::right_shift_magnitude[entry] != static_cast(bit_offset)) { + return false; + } + + if constexpr (BitWidth <= 16) { + const bool spills = bit_offset + BitWidth > lane_bytes * 8; + expected_has_spill |= spills; + for (usize byte = 0; byte < lane_bytes; ++byte) { + const u8 expected = spills && byte == 0 ? static_cast(first_byte + lane_bytes) : pernix::detail::inactive_lane; + if (table::spill_permute[base + byte] != expected) { + return false; + } + } + + const ShiftType expected_shift = spills ? static_cast((lane_bytes * 8) - bit_offset) : 0; + if (table::left_shift_for_spill[entry] != expected_shift) { + return false; + } + } + } + + if constexpr (BitWidth <= 16) { + if (table::has_spill != expected_has_spill) { + return false; + } + } + + return true; +} + +template +constexpr bool validate_all_bit_widths(std::index_sequence) { + return (validate_unpack_table(Indices + 1)>, static_cast(Indices + 1), VectorBytes>() && ...); +} + +static_assert(validate_all_bit_widths<16>(std::make_index_sequence<24>{})); +static_assert(validate_all_bit_widths<32>(std::make_index_sequence<24>{})); +static_assert(validate_all_bit_widths<64>(std::make_index_sequence<24>{})); +} // namespace + +TEST(UnpackTablesTest, PreservesSignedShiftCountsForNeonCompatibleConsumers) { + using byte_table = pernix::detail::unpack_table; + EXPECT_EQ(byte_table::right_shift[1], -5); + EXPECT_EQ(byte_table::right_shift_magnitude[1], 5); + EXPECT_EQ(byte_table::left_shift_for_spill[1], 3); + + using halfword_table = pernix::detail::unpack_table; + EXPECT_EQ(halfword_table::right_shift[1], -5); + EXPECT_EQ(halfword_table::right_shift_magnitude[1], 5); + EXPECT_EQ(halfword_table::left_shift_for_spill[1], 11); + + using word_table = pernix::detail::unpack_table; + EXPECT_EQ(word_table::right_shift[1], -3); + EXPECT_EQ(word_table::right_shift_magnitude[1], 3); +}