HADOOP-19855. Replace native zstd C bindings with zstd-jni library

BUILDING.txt

@@ -107,8 +107,6 @@ Optional packages:
   $ sudo apt-get install bzip2 libbz2-dev
 * Linux FUSE
   $ sudo apt-get install fuse libfuse-dev
-* ZStandard compression
-  $ sudo apt-get install libzstd1-dev
 * PMDK library for storage class memory(SCM) as HDFS cache backend
   Please refer to http://pmem.io/ and https://github.com/pmem/pmdk
 
@@ -196,29 +194,6 @@ Maven build goals:
     and it ignores the -Dsnappy.prefix option. If -Dsnappy.lib isn't given, the
     bundling and building will fail.
 
-
- ZStandard build options:
-
-   ZStandard is a compression library that can be utilized by the native code.
-   It is currently an optional component, meaning that Hadoop can be built with
-   or without this dependency.
-
-  * Use -Drequire.zstd to fail the build if libzstd.so is not found.
-    If this option is not specified and the zstd library is missing.
-
-  * Use -Dzstd.prefix to specify a nonstandard location for the libzstd
-    header files and library files. You do not need this option if you have
-    installed zstandard using a package manager.
-
-  * Use -Dzstd.lib to specify a nonstandard location for the libzstd library
-    files.  Similarly to zstd.prefix, you do not need this option if you have
-    installed using a package manager.
-
-  * Use -Dbundle.zstd to copy the contents of the zstd.lib directory into
-    the final tar file. This option requires that -Dzstd.lib is also given,
-    and it ignores the -Dzstd.prefix option. If -Dzstd.lib isn't given, the
-    bundling and building will fail.
-
  OpenSSL build options:
 
    OpenSSL includes a crypto library that can be utilized by the native code.
@@ -556,10 +531,6 @@ Building on Rocky Linux 8
 * Install optional dependencies (snappy-devel).
   $ sudo dnf --enablerepo=PowerTools install snappy-devel
 
-* Install optional dependencies (libzstd-devel).
-  $ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-  $ sudo dnf --enablerepo=epel install libzstd-devel
-
 * Install optional dependencies (isa-l).
   $ sudo dnf --enablerepo=PowerTools install nasm
   $ git clone https://github.com/intel/isa-l

dev-support/docker/Dockerfile_rockylinux_8

@@ -117,7 +117,6 @@ RUN pkg-resolver/install-maven.sh rockylinux:8
 RUN pkg-resolver/install-boost.sh rockylinux:8
 RUN pkg-resolver/install-spotbugs.sh rockylinux:8
 RUN pkg-resolver/install-protobuf.sh rockylinux:8
-RUN pkg-resolver/install-zstandard.sh rockylinux:8
 RUN pkg-resolver/install-intel-isa-l.sh rockylinux:8
 RUN pkg-resolver/install-common-pkgs.sh
 

dev-support/docker/pkg-resolver/packages.json

@@ -222,19 +222,15 @@
   },
   "zlib": {
     "debian:12": [
-      "libzstd-dev",
       "zlib1g-dev"
     ],
     "debian:13": [
-      "libzstd-dev",
       "zlib1g-dev"
     ],
     "ubuntu:noble": [
-      "libzstd-dev",
       "zlib1g-dev"
     ],
     "ubuntu:noble::arch64": [
-      "libzstd-dev",
       "zlib1g-dev"
     ],
     "rockylinux:8": [

hadoop-client-modules/hadoop-client-api/pom.xml

@@ -67,13 +67,17 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <!-- snappy-java is native library and cannot be relocated. So we explicitly exclude it
-         from shaded jar to prevent possible conflict. Make it as transitive dependency to
-         make the downstream pull it. -->
+    <!-- snappy-java/zstd-jni contains native library and cannot be relocated.
+         So we explicitly exclude it from shaded jar to prevent possible conflict.
+         Declare it as transitive dependency to make the downstream pull it. -->
     <dependency>
       <groupId>org.xerial.snappy</groupId>
       <artifactId>snappy-java</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.github.luben</groupId>
+      <artifactId>zstd-jni</artifactId>
+    </dependency>
   </dependencies>
   <profiles>
     <profile>
@@ -106,8 +110,9 @@
                       <include>org.apache.hadoop:*</include>
                     </includes>
                     <excludes>
-                      <!-- Leave snappy that includes native methods which cannot be relocated. -->
+                      <!-- Leave snappy/zstd-jni that includes native methods which cannot be relocated. -->
                       <exclude>org.xerial.snappy:*</exclude>
+                      <exclude>com.github.luben:zstd-jni</exclude>
                     </excludes>
                   </artifactSet>
                   <filters>
@@ -173,6 +178,9 @@
                         <exclude>com/sun/management/**/*</exclude>
                         <exclude>com/ibm/security/*</exclude>
                         <exclude>com/ibm/security/**/*</exclude>
+                        <!-- Exclude zstd-jni -->
+                        <exclude>com/github/luben/zstd/*</exclude>
+                        <exclude>com/github/luben/zstd/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>

hadoop-client-modules/hadoop-client-check-invariants/pom.xml

@@ -92,8 +92,9 @@
                     <exclude>org.glassfish.jersey:*</exclude>
                     <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
                     <exclude>org.bouncycastle:*</exclude>
-                    <!-- Leave snappy that includes native methods which cannot be relocated. -->
+                    <!-- Leave snappy/zstd-jni that includes native methods which cannot be relocated. -->
                     <exclude>org.xerial.snappy:*</exclude>
+                    <exclude>com.github.luben:zstd-jni</exclude>
                   </excludes>
                 </banTransitiveDependencies>
                 <banDuplicateClasses>

hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml

@@ -98,8 +98,9 @@
                     <exclude>com.google.code.findbugs:jsr305</exclude>
                     <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
                     <exclude>org.bouncycastle:*</exclude>
-                    <!-- Leave snappy that includes native methods which cannot be relocated. -->
+                    <!-- Leave snappy/zstd-jni that includes native methods which cannot be relocated. -->
                     <exclude>org.xerial.snappy:*</exclude>
+                    <exclude>com.github.luben:zstd-jni</exclude>
                     <exclude>org.ehcache:*</exclude>
                     <exclude>org.glassfish.jersey:*</exclude>
                   </excludes>

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -46,6 +46,11 @@
       <artifactId>snappy-java</artifactId>
       <scope>runtime</scope>
     </dependency>
+    <dependency>
+      <groupId>com.github.luben</groupId>
+      <artifactId>zstd-jni</artifactId>
+      <scope>runtime</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client-runtime</artifactId>
@@ -707,8 +712,9 @@
                       <!-- We need a filter that matches just those things that are included in the above artiacts -->
                       <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
                       <exclude>org.bouncycastle:*</exclude>
-                      <!-- Leave snappy that includes native methods which cannot be relocated. -->
+                      <!-- Leave snappy/zstd-jni that includes native methods which cannot be relocated. -->
                       <exclude>org.xerial.snappy:*</exclude>
+                      <exclude>com.github.luben:zstd-jni</exclude>
                       <exclude>org.glassfish.jersey:*</exclude>
                     </excludes>
                   </artifactSet>
@@ -947,6 +953,9 @@
                         <exclude>com/sun/management/**/*</exclude>
                         <exclude>com/ibm/security/*</exclude>
                         <exclude>com/ibm/security/**/*</exclude>
+                        <!-- Exclude zstd-jni -->
+                        <exclude>com/github/luben/zstd/*</exclude>
+                        <exclude>com/github/luben/zstd/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -66,6 +66,11 @@
       <artifactId>snappy-java</artifactId>
       <scope>runtime</scope>
     </dependency>
+    <dependency>
+      <groupId>com.github.luben</groupId>
+      <artifactId>zstd-jni</artifactId>
+      <scope>runtime</scope>
+    </dependency>
     <!-- This comes from our parent pom. If we don't expressly change it here to get included,
          downstream will get warnings at compile time. -->
     <dependency>
@@ -170,6 +175,7 @@
                       <exclude>org.bouncycastle:*</exclude>
                       <!-- Leave snappy that includes native methods which cannot be relocated. -->
                       <exclude>org.xerial.snappy:*</exclude>
+                      <exclude>com.github.luben:zstd-jni</exclude>
                       <!-- leave out kotlin classes -->
                       <exclude>org.jetbrains.kotlin:*</exclude>
                       <exclude>org.glassfish.jersey.test-framework:*</exclude>
@@ -320,6 +326,9 @@
                         <exclude>com/sun/management/**/*</exclude>
                         <exclude>com/ibm/security/*</exclude>
                         <exclude>com/ibm/security/**/*</exclude>
+                        <!-- Exclude zstd-jni -->
+                        <exclude>com/github/luben/zstd/*</exclude>
+                        <exclude>com/github/luben/zstd/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>

hadoop-common-project/hadoop-common/pom.xml

@@ -370,6 +370,11 @@
       <artifactId>lz4-java</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>com.github.luben</groupId>
+      <artifactId>zstd-jni</artifactId>
+      <scope>compile</scope>
+    </dependency>
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-api</artifactId>
@@ -696,10 +701,6 @@
       </activation>
       <properties>
         <require.bzip2>false</require.bzip2>
-        <zstd.prefix></zstd.prefix>
-        <zstd.lib></zstd.lib>
-        <zstd.include></zstd.include>
-        <require.zstd>false</require.zstd>
         <openssl.prefix></openssl.prefix>
         <openssl.lib></openssl.lib>
         <openssl.include></openssl.include>
@@ -749,10 +750,6 @@
                     <GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH>
                     <JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
                     <REQUIRE_BZIP2>${require.bzip2}</REQUIRE_BZIP2>
-                    <REQUIRE_ZSTD>${require.zstd}</REQUIRE_ZSTD>
-                    <CUSTOM_ZSTD_PREFIX>${zstd.prefix}</CUSTOM_ZSTD_PREFIX>
-                    <CUSTOM_ZSTD_LIB>${zstd.lib} </CUSTOM_ZSTD_LIB>
-                    <CUSTOM_ZSTD_INCLUDE>${zstd.include} </CUSTOM_ZSTD_INCLUDE>
                     <REQUIRE_ISAL>${require.isal} </REQUIRE_ISAL>
                     <CUSTOM_ISAL_PREFIX>${isal.prefix} </CUSTOM_ISAL_PREFIX>
                     <CUSTOM_ISAL_LIB>${isal.lib} </CUSTOM_ISAL_LIB>
@@ -807,11 +804,6 @@
         <require.isal>false</require.isal>
         <isal.prefix></isal.prefix>
         <isal.lib></isal.lib>
-        <zstd.prefix></zstd.prefix>
-        <zstd.lib></zstd.lib>
-        <zstd.include></zstd.include>
-        <require.zstd>false</require.zstd>
-        <bundle.zstd.in.bin>true</bundle.zstd.in.bin>
         <openssl.prefix></openssl.prefix>
         <openssl.lib></openssl.lib>
         <openssl.include></openssl.include>
@@ -961,10 +953,6 @@
                     <argument>/nologo</argument>
                     <argument>/p:Configuration=Release</argument>
                     <argument>/p:OutDir=${project.build.directory}/bin/</argument>
-                    <argument>/p:CustomZstdPrefix=${zstd.prefix}</argument>
-                    <argument>/p:CustomZstdLib=${zstd.lib}</argument>
-                    <argument>/p:CustomZstdInclude=${zstd.include}</argument>
-                    <argument>/p:RequireZstd=${require.zstd}</argument>
                     <argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
                     <argument>/p:CustomOpensslLib=${openssl.lib}</argument>
                     <argument>/p:CustomOpensslInclude=${openssl.include}</argument>
@@ -989,10 +977,6 @@
                     <argument>/nologo</argument>
                     <argument>/p:Configuration=Release</argument>
                     <argument>/p:OutDir=${project.build.directory}/bin/</argument>
-                    <argument>/p:CustomZstdPrefix=${zstd.prefix}</argument>
-                    <argument>/p:CustomZstdLib=${zstd.lib}</argument>
-                    <argument>/p:CustomZstdInclude=${zstd.include}</argument>
-                    <argument>/p:RequireZstd=${require.zstd}</argument>
                     <argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
                     <argument>/p:CustomOpensslLib=${openssl.lib}</argument>
                     <argument>/p:CustomOpensslInclude=${openssl.include}</argument>

hadoop-common-project/hadoop-common/src/CMakeLists.txt

@@ -67,33 +67,6 @@ else()
 endif()
 set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
 
-# Require zstandard
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
-hadoop_set_find_shared_library_version("1")
-find_library(ZSTD_LIBRARY
-    NAMES zstd
-    PATHS ${CUSTOM_ZSTD_PREFIX} ${CUSTOM_ZSTD_PREFIX}/lib
-          ${CUSTOM_ZSTD_PREFIX}/lib64 ${CUSTOM_ZSTD_LIB})
-SET(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
-find_path(ZSTD_INCLUDE_DIR
-    NAMES zstd.h
-    PATHS ${CUSTOM_ZSTD_PREFIX} ${CUSTOM_ZSTD_PREFIX}/include
-          ${CUSTOM_ZSTD_INCLUDE})
-if (ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
-    GET_FILENAME_COMPONENT(HADOOP_ZSTD_LIBRARY ${ZSTD_LIBRARY} NAME)
-    set(ZSTD_SOURCE_FILES
-        "${SRC}/io/compress/zstd/ZStandardCompressor.c"
-        "${SRC}/io/compress/zstd/ZStandardDecompressor.c")
-    set(REQUIRE_ZSTD ${REQUIRE_ZSTD}) # Stop warning about unused variable.
-        message(STATUS "Found ZStandard: ${ZSTD_LIBRARY}")
-else ()
-    set(ZSTD_INCLUDE_DIR "")
-    set(ZSTD_SOURCE_FILES "")
-    IF(REQUIRE_ZSTD)
-        MESSAGE(FATAL_ERROR "Required zstandard library could not be found.  ZSTD_LIBRARY=${ZSTD_LIBRARY}, ZSTD_INCLUDE_DIR=${ZSTD_INCLUDE_DIR}, CUSTOM_ZSTD_INCLUDE_DIR=${CUSTOM_ZSTD_INCLUDE_DIR}, CUSTOM_ZSTD_PREFIX=${CUSTOM_ZSTD_PREFIX}, CUSTOM_ZSTD_INCLUDE=${CUSTOM_ZSTD_INCLUDE}")
-    ENDIF(REQUIRE_ZSTD)
-endif ()
-
 #Require ISA-L
 set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
 hadoop_set_find_shared_library_version("2")
@@ -229,7 +202,6 @@ include_directories(
     ${ZLIB_INCLUDE_DIRS}
     ${BZIP2_INCLUDE_DIR}
     ${ISAL_INCLUDE_DIR}
-    ${ZSTD_INCLUDE_DIR}
     ${OPENSSL_INCLUDE_DIR}
     ${SRC}/util
 )
@@ -239,7 +211,6 @@ set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 hadoop_add_dual_library(hadoop
     main/native/src/exception.c
     ${ISAL_SOURCE_FILES}
-    ${ZSTD_SOURCE_FILES}
     ${OPENSSL_SOURCE_FILES}
     ${SRC}/io/compress/zlib/ZlibCompressor.c
     ${SRC}/io/compress/zlib/ZlibDecompressor.c

hadoop-common-project/hadoop-common/src/config.h.cmake

@@ -21,7 +21,6 @@
 #cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
 #cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@"
 #cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
-#cmakedefine HADOOP_ZSTD_LIBRARY "@HADOOP_ZSTD_LIBRARY@"
 #cmakedefine HADOOP_OPENSSL_LIBRARY "@HADOOP_OPENSSL_LIBRARY@"
 #cmakedefine HADOOP_ISAL_LIBRARY "@HADOOP_ISAL_LIBRARY@"
 #cmakedefine HADOOP_PMDK_LIBRARY "@HADOOP_PMDK_LIBRARY@"