From ea914bba6e2f770811512e996a6d66875ab3fd6e Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 21 Jul 2026 09:24:47 +0100 Subject: [PATCH 01/20] [VL] Fix gpu docker image the adapters image only installed minimal dependency libs, this patch installed the missing package for building gluten-gpu support Signed-off-by: Yuan --- dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf b/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf index 675a601c198..913f9ac7eee 100644 --- a/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf +++ b/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf @@ -31,7 +31,7 @@ ENV CXX=/opt/rh/gcc-toolset-14/root/bin/g++ ENV LD_LIBRARY_PATH=/opt/gluten/ep/build-velox/build/velox_ep/_build/release/_deps/curl-build/lib:$LD_LIBRARY_PATH # Install dependencies and cudf related libraries to /usr -RUN yum install -y sudo patch perl java-${JAVA_VERSION}-openjdk-devel && \ +RUN yum install -y sudo patch perl java-${JAVA_VERSION}-openjdk-devel cuda-nvtx-12-9 && \ ln -sf /usr/local/bin/cmake /usr/bin && \ git clone --depth=1 https://github.com/apache/gluten /opt/gluten && \ cd /opt/gluten && \ From a690a49da8745f3fd43a2e880bf6b0c009a29d79 Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 08:46:14 +0100 Subject: [PATCH 02/20] fix Signed-off-by: Yuan --- ep/build-velox/src/get-velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 72865dc94a6..5afc1af735a 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -65,7 +65,7 @@ if [ "$VELOX_HOME" == "" ]; then fi function process_setup_ubuntu { - sed -i "s|run_and_time install_arrow||g" scripts/setup-centos9.sh + sed -i "s|run_and_time install_arrow||g" scripts/setup-ubuntu.sh echo "Using setup script from Velox" } From e870ff546682ced545735abc59005a562908138e Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 08:47:02 +0100 Subject: [PATCH 03/20] adding ubuntu24 Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 43bac7fc6b7..a283b04f460 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -73,7 +73,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ "ubuntu:22.04" ] + os: [ "ubuntu:22.04", "ubuntu:22.04" ] if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-22.04 container: ${{ matrix.os }} From 1fb3272e9147ac7c051acd5ae322e4904fa2c0d4 Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 08:48:01 +0100 Subject: [PATCH 04/20] adding basic tpc test Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index a283b04f460..eca2480012b 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -111,6 +111,14 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 export VELOX_BUILD_SHARED=ON cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 + cd $GITHUB_WORKSPACE/tools/gluten-it + $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 + - name: Run TPC-H + run: | + echo "JAVA_HOME: $JAVA_HOME" + cd $GITHUB_WORKSPACE/tools/gluten-it + $TPCH_TEST + weekly-build-on-openeuler: strategy: From 4d0d4b1c41aae91d348b182499aab62f9e89f5fa Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 08:51:09 +0100 Subject: [PATCH 05/20] fix Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index eca2480012b..a7b35919a9c 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -68,12 +68,19 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH export VELOX_BUILD_SHARED=ON cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 + cd $GITHUB_WORKSPACE/tools/gluten-it + $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 + - name: Run TPC-H + run: | + echo "JAVA_HOME: $JAVA_HOME" + cd $GITHUB_WORKSPACE/tools/gluten-it + $TPCH_TEST weekly-build-on-ubuntu: strategy: fail-fast: false matrix: - os: [ "ubuntu:22.04", "ubuntu:22.04" ] + os: [ "ubuntu:22.04", "ubuntu:24.04" ] if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-22.04 container: ${{ matrix.os }} From 4699badcc6c15a3f23898b855661db4b5d44790e Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 09:07:09 +0100 Subject: [PATCH 06/20] fix cmake install Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index a7b35919a9c..64f57010e7f 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -92,9 +92,12 @@ jobs: # To avoid the prompt for region selection during installing tzdata. export DEBIAN_FRONTEND=noninteractive export INSTALL_PREFIX=/usr/ - apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential cmake openjdk-8-jdk python3-pip - pip3 install cmake==3.31.4 + apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential openjdk-8-jdk python3-pip cd /tmp + wget https://github.com/Kitware/CMake/releases/download/v3.31.12/cmake-3.31.12-linux-$(uname -m).sh; \ + chmod +x cmake-3.31.12-linux-$(uname -m).sh; \ + ./cmake-3.31.12-linux-$(uname -m).sh --skip-license --prefix=/usr/local; \ + rm -f cmake-3.31.12-linux-$(uname -m).sh; wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz tar -xzf gflags.tar.gz && cd gflags-2.2.2 cmake -B build \ From 94c3260060eb07c88814ecb3d90cc0acd9367ebe Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 14:33:06 +0100 Subject: [PATCH 07/20] fix java home Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 64f57010e7f..a2eeee6f5b5 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -72,6 +72,7 @@ jobs: $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 - name: Run TPC-H run: | + export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 echo "JAVA_HOME: $JAVA_HOME" cd $GITHUB_WORKSPACE/tools/gluten-it $TPCH_TEST @@ -125,6 +126,7 @@ jobs: $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 - name: Run TPC-H run: | + export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 echo "JAVA_HOME: $JAVA_HOME" cd $GITHUB_WORKSPACE/tools/gluten-it $TPCH_TEST From a041eb2027302c59224684d011ab2fb73d280cb9 Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 14:44:10 +0100 Subject: [PATCH 08/20] fix mac build Signed-off-by: Yuan --- ep/build-velox/src/get-velox.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 5afc1af735a..063d8e54dee 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -96,6 +96,7 @@ function process_setup_tencentos32 { # independent of this. Header-search isolation from /usr/local is handled by # SDKROOT exported in builddeps-veloxbe.sh / build-velox.sh. function process_setup_macos { + sed -i '' 's/-DFMT_TEST=OFF/-DFMT_TEST=OFF -DFMT_USE_STD_FORMAT=OFF/' scripts/setup-macos.sh if ! grep -Fq 'FOLLY_USE_JEMALLOC=OFF' scripts/setup-common.sh; then sed -i '' 's/local FOLLY_FLAGS=(/local FOLLY_FLAGS=(-DFOLLY_USE_JEMALLOC=OFF /' scripts/setup-common.sh fi From 8e0af66e3b7493c1cacd0b0dfb41724520b82f14 Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 16:00:03 +0100 Subject: [PATCH 09/20] adding cache for weekly job Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 129 +++++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 6 deletions(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index a2eeee6f5b5..ce87a51d42c 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -25,9 +25,10 @@ on: env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - TPCH_TEST: "env GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare + TPCH_TEST: "env GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1" INSTALL_PREFIX: /usr/local + CCACHE_DIR: "${{ github.workspace }}/.ccache" concurrency: @@ -47,6 +48,22 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container + run: | + case "$(uname -m)" in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + esac + curl -fsSL "https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz" \ + | tar -xz -C /usr/local --strip-components=1 "gh_2.63.2_linux_${ARCH}/bin/gh" + curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}" + chmod +x /usr/local/bin/jq + gh --version && jq --version + - name: Get Ccache from Apache Stash + uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-centos-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }} - name: build run: | if [ "${{ matrix.os }}" = "centos:8" ]; then @@ -63,13 +80,22 @@ jobs: dnf install -y epel-release sudo dnf dnf install -y --setopt=install_weak_deps=False gcc-toolset-12 source /opt/rh/gcc-toolset-12/enable || exit 1 - yum install -y java-1.8.0-openjdk-devel patch wget git perl + yum install -y java-1.8.0-openjdk-devel patch wget git perl ccache export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ export PATH=$JAVA_HOME/bin:$PATH + export CCACHE_MAXSIZE=1G + mkdir -p $CCACHE_DIR export VELOX_BUILD_SHARED=ON cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 + - name: Save Ccache to Apache Stash + if: always() + uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-centos-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }} + include-hidden-files: true - name: Run TPC-H run: | export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 @@ -87,13 +113,30 @@ jobs: container: ${{ matrix.os }} steps: - uses: actions/checkout@v4 + - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container + run: | + case "$(uname -m)" in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + esac + apt-get update -y && apt-get install -y curl + curl -fsSL "https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz" \ + | tar -xz -C /usr/local --strip-components=1 "gh_2.63.2_linux_${ARCH}/bin/gh" + curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}" + chmod +x /usr/local/bin/jq + gh --version && jq --version + - name: Get Ccache from Apache Stash + uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-ubuntu-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }} - name: build run: | # sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list # To avoid the prompt for region selection during installing tzdata. export DEBIAN_FRONTEND=noninteractive export INSTALL_PREFIX=/usr/ - apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential openjdk-8-jdk python3-pip + apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential openjdk-8-jdk python3-pip ccache cd /tmp wget https://github.com/Kitware/CMake/releases/download/v3.31.12/cmake-3.31.12-linux-$(uname -m).sh; \ chmod +x cmake-3.31.12-linux-$(uname -m).sh; \ @@ -120,10 +163,19 @@ jobs: cmake . && make install -j apt autoremove -y export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + export CCACHE_MAXSIZE=1G + mkdir -p $CCACHE_DIR export VELOX_BUILD_SHARED=ON cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 + - name: Save Ccache to Apache Stash + if: always() + uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-ubuntu-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }} + include-hidden-files: true - name: Run TPC-H run: | export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 @@ -144,16 +196,41 @@ jobs: - uses: actions/checkout@v4 - name: Install dependencies run: | - dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl + dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl ccache curl echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV + - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container + run: | + case "$(uname -m)" in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + esac + curl -fsSL "https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz" \ + | tar -xz -C /usr/local --strip-components=1 "gh_2.63.2_linux_${ARCH}/bin/gh" + curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}" + chmod +x /usr/local/bin/jq + gh --version && jq --version + - name: Get Ccache from Apache Stash + uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-openeuler-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }} - name: Build run: | echo "JAVA_HOME: $JAVA_HOME" + export CCACHE_MAXSIZE=1G + mkdir -p $CCACHE_DIR cd $GITHUB_WORKSPACE/ ./dev/package.sh --spark_version=3.5 cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 cp $GITHUB_WORKSPACE/package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar package/target/lib/ + - name: Save Ccache to Apache Stash + if: always() + uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-openeuler-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }} + include-hidden-files: true - name: Run TPC-H run: | echo "JAVA_HOME: $JAVA_HOME" @@ -173,16 +250,41 @@ jobs: - uses: actions/checkout@v4 - name: Install dependencies run: | - dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl + dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl curl echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV + - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container + run: | + case "$(uname -m)" in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + esac + curl -fsSL "https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz" \ + | tar -xz -C /usr/local --strip-components=1 "gh_2.63.2_linux_${ARCH}/bin/gh" + curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}" + chmod +x /usr/local/bin/jq + gh --version && jq --version + - name: Get Ccache from Apache Stash + uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-openeuler-vcpkg-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }} - name: Build run: | echo "JAVA_HOME: $JAVA_HOME" + export CCACHE_MAXSIZE=1G + mkdir -p $CCACHE_DIR cd $GITHUB_WORKSPACE/ ./dev/vcpkg/setup-build-depends.sh ./dev/package-vcpkg.sh --build_arrow=ON --spark_version=3.5 cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 + - name: Save Ccache to Apache Stash + if: always() + uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-openeuler-vcpkg-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }} + include-hidden-files: true - name: Run TPC-H run: | echo "JAVA_HOME: $JAVA_HOME" @@ -205,5 +307,20 @@ jobs: java-version: 17 - name: Install Dependencies run: brew install wget pkgconfig abseil ccache + - name: Get Ccache from Apache Stash + uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-macos-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }} - name: Build - run: ./dev/buildbundle-veloxbe.sh --run_setup_script=ON --build_arrow=ON --build_tests=ON --build_benchmarks=ON --spark_version=3.5 + run: | + export CCACHE_MAXSIZE=1G + mkdir -p $CCACHE_DIR + ./dev/buildbundle-veloxbe.sh --run_setup_script=ON --build_arrow=ON --build_tests=ON --build_benchmarks=ON --spark_version=3.5 + - name: Save Ccache to Apache Stash + if: always() + uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-weekly-macos-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }} + include-hidden-files: true From 31f594e1a70cd4a93ad795f19e30df4a78dc0178 Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 16:52:04 +0100 Subject: [PATCH 10/20] fix package Signed-off-by: Yuan --- dev/build-thirdparty.sh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/dev/build-thirdparty.sh b/dev/build-thirdparty.sh index 077999ccce2..62efe4efe9e 100755 --- a/dev/build-thirdparty.sh +++ b/dev/build-thirdparty.sh @@ -39,7 +39,7 @@ function process_setup_ubuntu_2204 { function process_setup_centos_9 { cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libsodium.so.23} $THIRDPARTY_LIB/ cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/ - cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/ + cp /usr/local/lib/libgflags.so.2.3 $THIRDPARTY_LIB/ find /usr/local/lib /usr/local/lib64 -name "libglog.so.1" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null find /usr/local/lib /usr/local/lib64 -name "libgeos.so.3.10.7" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null } @@ -47,29 +47,23 @@ function process_setup_centos_9 { function process_setup_centos_8 { cp /usr/lib64/{libre2.so.0,libdouble-conversion.so.3,libevent-2.1.so.6,libdwarf.so.1,libicudata.so.60,libicui18n.so.60,libicuuc.so.60,libsodium.so.23} $THIRDPARTY_LIB/ cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/ - cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1,libgeos.so.3.10.7} $THIRDPARTY_LIB/ -} - -function process_setup_centos_7 { - cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.0,libgeos.so.3.10.7} $THIRDPARTY_LIB/ - cp /usr/lib64/{libdouble-conversion.so.1,libevent-2.0.so.5,libzstd.so.1,libntlm.so.0,libgsasl.so.7,liblz4.so.1} $THIRDPARTY_LIB/ - cp /usr/local/lib/{libre2.so.10,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_regex.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/ + cp /usr/local/lib64/{libgflags.so.2.3,libglog.so.1,libgeos.so.3.10.7} $THIRDPARTY_LIB/ } function process_setup_debian_11 { - cp /usr/lib/x86_64-linux-gnu/{libre2.so.9,libthrift-0.13.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.2,libglog.so.0,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.67,libicuuc.so.67,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap_r-2.4.so.2,liblber-2.4.so.2,libbrotlidec.so.1,libicudata.so.67,libsasl2.so.2,libbrotlicommon.so.1} $THIRDPARTY_LIB/ + cp /usr/lib/x86_64-linux-gnu/{libre2.so.9,libthrift-0.13.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.3,libglog.so.0,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.67,libicuuc.so.67,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap_r-2.4.so.2,liblber-2.4.so.2,libbrotlidec.so.1,libicudata.so.67,libsasl2.so.2,libbrotlicommon.so.1} $THIRDPARTY_LIB/ cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_regex.so.1.84.0} $THIRDPARTY_LIB/ } function process_setup_debian_12 { - cp /usr/lib/x86_64-linux-gnu/{libthrift-0.17.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.2,libglog.so.1,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.72,libicuuc.so.72,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap-2.5.so.0,liblber-2.5.so.0,libbrotlidec.so.1,libicudata.so.72,libsasl2.so.2,libbrotlicommon.so.1,libcrypto.so.3,libssl.so.3,libgssapi_krb5.so.2,libkrb5.so.3,libk5crypto.so.3,libkrb5support.so.0,libkeyutils.so.1} $THIRDPARTY_LIB/ + cp /usr/lib/x86_64-linux-gnu/{libthrift-0.17.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.3,libglog.so.1,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.72,libicuuc.so.72,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap-2.5.so.0,liblber-2.5.so.0,libbrotlidec.so.1,libicudata.so.72,libsasl2.so.2,libbrotlicommon.so.1,libcrypto.so.3,libssl.so.3,libgssapi_krb5.so.2,libkrb5.so.3,libk5crypto.so.3,libkrb5support.so.0,libkeyutils.so.1} $THIRDPARTY_LIB/ cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_regex.so.1.84.0} $THIRDPARTY_LIB/ } function process_setup_openeuler_24 { cp /usr/lib64/{libre2.so.11,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libgsasl.so.7,libicudata.so.74,libicui18n.so.74,libicuuc.so.74,libidn.so.12,libntlm.so.0,libsodium.so.26} $THIRDPARTY_LIB/ cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/ - cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1} $THIRDPARTY_LIB/ + cp /usr/local/lib64/{libgflags.so.2.3,libglog.so.1} $THIRDPARTY_LIB/ } if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then From 704c2daeb135538eec348ea99b53aac9a768641d Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 16:53:13 +0100 Subject: [PATCH 11/20] fix package Signed-off-by: Yuan --- dev/build-thirdparty.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dev/build-thirdparty.sh b/dev/build-thirdparty.sh index 62efe4efe9e..051343493cd 100755 --- a/dev/build-thirdparty.sh +++ b/dev/build-thirdparty.sh @@ -31,7 +31,12 @@ function process_setup_ubuntu_2004 { } function process_setup_ubuntu_2204 { - cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libgflags.so.2.2,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70} $THIRDPARTY_LIB/ + cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libgflags.so.2.3,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70} $THIRDPARTY_LIB/ + find /usr/ -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null + find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null +} +function process_setup_ubuntu_2404 { + cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libgflags.so.2.3,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70} $THIRDPARTY_LIB/ find /usr/ -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null } @@ -71,6 +76,8 @@ if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then process_setup_ubuntu_2004 elif [ "$VERSION" == "22.04" ]; then process_setup_ubuntu_2204 + elif [ "$VERSION" == "24.04" ]; then + process_setup_ubuntu_2404 fi elif [ "$LINUX_OS" == "centos" ]; then if [ "$VERSION" == "9" ]; then From 5f735bc90952ef67c8c4b03d6baab0109a8cc350 Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 24 Jul 2026 17:37:25 +0100 Subject: [PATCH 12/20] fix Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index ce87a51d42c..a457679f018 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -98,7 +98,7 @@ jobs: include-hidden-files: true - name: Run TPC-H run: | - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk echo "JAVA_HOME: $JAVA_HOME" cd $GITHUB_WORKSPACE/tools/gluten-it $TPCH_TEST From 4039c10c510f4c133725e573e9e0870e0bfe4726 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Jul 2026 09:33:20 +0100 Subject: [PATCH 13/20] fix Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index a457679f018..add8a1662ea 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -48,8 +48,16 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container + - name: Install Stash restore dependencies # the stash action needs gh, jq, and python3 (>=3.7), absent from this container run: | + if [ "${{ matrix.os }}" = "centos:8" ]; then + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true + # centos:8 ships python3.6 by default; install python39 from AppStream and override the symlink + dnf install -y python39 && ln -sf /usr/bin/python3.9 /usr/bin/python3 + else + dnf install -y python3 + fi case "$(uname -m)" in x86_64) ARCH=amd64 ;; aarch64) ARCH=arm64 ;; @@ -113,13 +121,13 @@ jobs: container: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container + - name: Install Stash restore dependencies # the stash action needs gh, jq, and python3, absent from this container run: | case "$(uname -m)" in x86_64) ARCH=amd64 ;; aarch64) ARCH=arm64 ;; esac - apt-get update -y && apt-get install -y curl + apt-get update -y && apt-get install -y curl python3 curl -fsSL "https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz" \ | tar -xz -C /usr/local --strip-components=1 "gh_2.63.2_linux_${ARCH}/bin/gh" curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}" @@ -196,7 +204,7 @@ jobs: - uses: actions/checkout@v4 - name: Install dependencies run: | - dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl ccache curl + dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl ccache curl python3 echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container run: | @@ -250,7 +258,7 @@ jobs: - uses: actions/checkout@v4 - name: Install dependencies run: | - dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl curl + dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl curl python3 echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container run: | From a1a70b37663e0cfe3bda36363dbc4976c47c6eef Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Jul 2026 10:30:04 +0100 Subject: [PATCH 14/20] fix macos Signed-off-by: Yuan --- ep/build-velox/src/build-velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/build-velox.sh b/ep/build-velox/src/build-velox.sh index 3e279bf6808..08ea2230c40 100755 --- a/ep/build-velox/src/build-velox.sh +++ b/ep/build-velox/src/build-velox.sh @@ -120,7 +120,7 @@ function compile { # dependency headers (abseil's __is_trivially_relocatable, arrow's vendored # date.h literal operators) to non-fatal system-header warnings under # -Werror on recent clang. - CXX_FLAGS="$CXX_FLAGS -isystem ${INSTALL_PREFIX}/include" + CXX_FLAGS="$CXX_FLAGS ${INSTALL_PREFIX}/include -isystem " fi fi From 859511be2e4e5afb6f6ff509acb600f2ab8e6e86 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Jul 2026 10:59:00 +0100 Subject: [PATCH 15/20] fix Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index add8a1662ea..956b2ad2c14 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -97,6 +97,9 @@ jobs: cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 + - name: Clean Ccache tmp before save + if: always() + run: rm -rf '${{ env.CCACHE_DIR }}/tmp' - name: Save Ccache to Apache Stash if: always() uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 @@ -104,6 +107,7 @@ jobs: path: '${{ env.CCACHE_DIR }}' key: ccache-weekly-centos-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }} include-hidden-files: true + retention-days: '7' - name: Run TPC-H run: | export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk @@ -177,6 +181,9 @@ jobs: cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 + - name: Clean Ccache tmp before save + if: always() + run: rm -rf '${{ env.CCACHE_DIR }}/tmp' - name: Save Ccache to Apache Stash if: always() uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 @@ -184,6 +191,7 @@ jobs: path: '${{ env.CCACHE_DIR }}' key: ccache-weekly-ubuntu-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }} include-hidden-files: true + retention-days: '7' - name: Run TPC-H run: | export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 @@ -232,6 +240,9 @@ jobs: cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 cp $GITHUB_WORKSPACE/package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar package/target/lib/ + - name: Clean Ccache tmp before save + if: always() + run: rm -rf '${{ env.CCACHE_DIR }}/tmp' - name: Save Ccache to Apache Stash if: always() uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 @@ -239,6 +250,7 @@ jobs: path: '${{ env.CCACHE_DIR }}' key: ccache-weekly-openeuler-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }} include-hidden-files: true + retention-days: '7' - name: Run TPC-H run: | echo "JAVA_HOME: $JAVA_HOME" @@ -286,6 +298,9 @@ jobs: ./dev/package-vcpkg.sh --build_arrow=ON --spark_version=3.5 cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 + - name: Clean Ccache tmp before save + if: always() + run: rm -rf '${{ env.CCACHE_DIR }}/tmp' - name: Save Ccache to Apache Stash if: always() uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 @@ -293,6 +308,7 @@ jobs: path: '${{ env.CCACHE_DIR }}' key: ccache-weekly-openeuler-vcpkg-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }} include-hidden-files: true + retention-days: '7' - name: Run TPC-H run: | echo "JAVA_HOME: $JAVA_HOME" @@ -325,6 +341,9 @@ jobs: export CCACHE_MAXSIZE=1G mkdir -p $CCACHE_DIR ./dev/buildbundle-veloxbe.sh --run_setup_script=ON --build_arrow=ON --build_tests=ON --build_benchmarks=ON --spark_version=3.5 + - name: Clean Ccache tmp before save + if: always() + run: rm -rf '${{ env.CCACHE_DIR }}/tmp' - name: Save Ccache to Apache Stash if: always() uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 @@ -332,3 +351,4 @@ jobs: path: '${{ env.CCACHE_DIR }}' key: ccache-weekly-macos-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }} include-hidden-files: true + retention-days: '7' From 4860c2fadfbd5b9eea448afa1d1d5a013932a3de Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Jul 2026 15:28:55 +0100 Subject: [PATCH 16/20] fix ubuntu 22 Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 956b2ad2c14..ec93d85455b 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -154,8 +154,8 @@ jobs: chmod +x cmake-3.31.12-linux-$(uname -m).sh; \ ./cmake-3.31.12-linux-$(uname -m).sh --skip-license --prefix=/usr/local; \ rm -f cmake-3.31.12-linux-$(uname -m).sh; - wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz - tar -xzf gflags.tar.gz && cd gflags-2.2.2 + wget https://github.com/gflags/gflags/archive/v2.3.0.tar.gz -O gflags.tar.gz + tar -xzf gflags.tar.gz && cd gflags-2.3.0 cmake -B build \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DBUILD_SHARED_LIBS=ON \ From a9f9c18f514844e93dd725bccde8d63550b3844d Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Jul 2026 15:34:47 +0100 Subject: [PATCH 17/20] fix ubuntu 24 Signed-off-by: Yuan --- dev/build-thirdparty.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/build-thirdparty.sh b/dev/build-thirdparty.sh index 051343493cd..26d07ad7cc9 100755 --- a/dev/build-thirdparty.sh +++ b/dev/build-thirdparty.sh @@ -36,7 +36,8 @@ function process_setup_ubuntu_2204 { find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null } function process_setup_ubuntu_2404 { - cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libgflags.so.2.3,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70} $THIRDPARTY_LIB/ + cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.10,libdouble-conversion.so.3,libglog.so.1,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libnghttp2.so.14,libldap.so.2,liblber.so.2,librtmp.so.1,libsasl2.so.2,libssh.so.4,libsodium.so.23,libssl.so.3,libcrypto.so.3,libgssapi_krb5.so.2,libkrb5.so.3,libk5crypto.so.3,libkrb5support.so.0,libkeyutils.so.1,libbrotlidec.so.1,libbrotlicommon.so.1,libidn2.so.0} $THIRDPARTY_LIB/ + cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/ find /usr/ -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null } From 776bfda4594617ad623e12b9e4bfdcac56b88bde Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 27 Jul 2026 17:17:05 +0100 Subject: [PATCH 18/20] fix package Signed-off-by: Yuan --- ep/build-velox/src/build-velox.sh | 6 +++++- ep/build-velox/src/get-velox.sh | 1 - ep/build-velox/src/setup-centos8.sh | 2 +- ep/build-velox/src/setup-openeuler24.sh | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ep/build-velox/src/build-velox.sh b/ep/build-velox/src/build-velox.sh index 08ea2230c40..0c9e75b9efe 100755 --- a/ep/build-velox/src/build-velox.sh +++ b/ep/build-velox/src/build-velox.sh @@ -120,7 +120,7 @@ function compile { # dependency headers (abseil's __is_trivially_relocatable, arrow's vendored # date.h literal operators) to non-fatal system-header warnings under # -Werror on recent clang. - CXX_FLAGS="$CXX_FLAGS ${INSTALL_PREFIX}/include -isystem " + CXX_FLAGS="$CXX_FLAGS -isystem ${INSTALL_PREFIX}/include" fi fi @@ -134,6 +134,10 @@ function compile { COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PREFIX_PATH=/usr/local" COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PATH=/usr/local\;/usr/local/include\;/usr/local/lib\;/usr/local/lib/cmake" COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_SYSTEM_IGNORE_PATH=/usr/local\;/usr/local/include\;/usr/local/lib\;/usr/local/lib/cmake" + # Force fmt to build from source via FetchContent: Homebrew's fmt config file + # at /opt/homebrew/lib/cmake/fmt/ wins the AUTO find_package probe ahead of + # INSTALL_PREFIX, producing a version mismatch. BUNDLED skips find_package. + COMPILE_OPTION="$COMPILE_OPTION -Dfmt_SOURCE=BUNDLED" fi if [ $BUILD_TEST_UTILS == "ON" ]; then COMPILE_OPTION="$COMPILE_OPTION -DVELOX_BUILD_TEST_UTILS=ON" diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 063d8e54dee..5afc1af735a 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -96,7 +96,6 @@ function process_setup_tencentos32 { # independent of this. Header-search isolation from /usr/local is handled by # SDKROOT exported in builddeps-veloxbe.sh / build-velox.sh. function process_setup_macos { - sed -i '' 's/-DFMT_TEST=OFF/-DFMT_TEST=OFF -DFMT_USE_STD_FORMAT=OFF/' scripts/setup-macos.sh if ! grep -Fq 'FOLLY_USE_JEMALLOC=OFF' scripts/setup-common.sh; then sed -i '' 's/local FOLLY_FLAGS=(/local FOLLY_FLAGS=(-DFOLLY_USE_JEMALLOC=OFF /' scripts/setup-common.sh fi diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index 3aa6f922a9b..95b0e5586fb 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -102,7 +102,7 @@ function install_xxhash { function install_gflags { # Remove an older version if present. dnf remove -y gflags - wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags + wget_and_untar https://github.com/gflags/gflags/archive/v2.3.0.tar.gz gflags cmake_install_dir gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 } diff --git a/ep/build-velox/src/setup-openeuler24.sh b/ep/build-velox/src/setup-openeuler24.sh index 80a66917b22..39fd196e73a 100755 --- a/ep/build-velox/src/setup-openeuler24.sh +++ b/ep/build-velox/src/setup-openeuler24.sh @@ -83,7 +83,7 @@ function install_velox_deps_from_dnf { function install_gflags { # Remove an older version if present. dnf remove -y gflags - wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags + wget_and_untar https://github.com/gflags/gflags/archive/v2.3.0.tar.gz gflags cmake_install_dir gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 } From 35a61c3cdde38c989a05a750e950bf37c76f5b47 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 28 Jul 2026 02:21:10 +0100 Subject: [PATCH 19/20] fix Signed-off-by: Yuan --- .../org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala | 2 +- .../org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala | 2 +- .../org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala | 2 +- .../org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala | 2 +- .../org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala | 2 +- .../apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala | 2 +- .../apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala | 2 +- dev/build-thirdparty.sh | 5 +++-- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala index 64db4f23f83..a7d192b1f4d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala @@ -38,7 +38,7 @@ class SharedLibraryLoaderCentos7 extends SharedLibraryLoader { loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") loader.loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so") loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so") - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so") loader.loadAndCreateLink("libglog.so.0", "libglog.so") loader.loadAndCreateLink("libre2.so.10", "libre2.so") loader.loadAndCreateLink("libzstd.so.1", "libzstd.so") diff --git a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala index 02faf8649fa..d6b4b77ce8c 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala @@ -42,7 +42,7 @@ class SharedLibraryLoaderCentos8 extends SharedLibraryLoader { loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so") - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so") loader.loadAndCreateLink("libglog.so.1", "libglog.so") loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so") loader.loadAndCreateLink("libre2.so.0", "libre2.so") diff --git a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala index cfb9ff17eff..3862ece676a 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala @@ -38,7 +38,7 @@ class SharedLibraryLoaderCentos9 extends SharedLibraryLoader { loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so") - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so") loader.loadAndCreateLink("libglog.so.1", "libglog.so") loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so") loader.loadAndCreateLink("libre2.so.9", "libre2.so") diff --git a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala index f06589eeedc..f2d8a224ebb 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala @@ -33,7 +33,7 @@ class SharedLibraryLoaderDebian11 extends SharedLibraryLoader { loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so") loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so") loader.loadAndCreateLink("libunwind.so.8", "libunwind.so") - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so") loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so") loader.loadAndCreateLink("librtmp.so.1", "librtmp.so") loader.loadAndCreateLink("libssh2.so.1", "libssh2.so") diff --git a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala index 2ce752adf7c..642402392a6 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala @@ -39,7 +39,7 @@ class SharedLibraryLoaderDebian12 extends SharedLibraryLoader { loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so") loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so") loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so") - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so") loader.loadAndCreateLink("libunwind.so.8", "libunwind.so") loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so") loader.loadAndCreateLink("librtmp.so.1", "librtmp.so") diff --git a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala index 0705539c40d..a1599945480 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala @@ -36,7 +36,7 @@ class SharedLibraryLoaderOpenEuler2403 extends SharedLibraryLoader { loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so") - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so") loader.loadAndCreateLink("libglog.so.1", "libglog.so") loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so") loader.loadAndCreateLink("libidn.so.12", "libidn.so") diff --git a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala index 4786ee67108..14976c4913e 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala @@ -38,7 +38,7 @@ class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader { loader.loadAndCreateLink("libcurl.so.4", "libcurl.so") loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so") - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so") loader.loadAndCreateLink("libunwind.so.8", "libunwind.so") loader.loadAndCreateLink("libglog.so.0", "libglog.so") loader.loadAndCreateLink("libidn.so.12", "libidn.so") diff --git a/dev/build-thirdparty.sh b/dev/build-thirdparty.sh index 26d07ad7cc9..a3503e76456 100755 --- a/dev/build-thirdparty.sh +++ b/dev/build-thirdparty.sh @@ -31,13 +31,14 @@ function process_setup_ubuntu_2004 { } function process_setup_ubuntu_2204 { - cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libgflags.so.2.3,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70} $THIRDPARTY_LIB/ + cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70} $THIRDPARTY_LIB/ + cp /usr/local/lib/libgflags.so.2.3 $THIRDPARTY_LIB/ find /usr/ -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null } function process_setup_ubuntu_2404 { cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.10,libdouble-conversion.so.3,libglog.so.1,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libnghttp2.so.14,libldap.so.2,liblber.so.2,librtmp.so.1,libsasl2.so.2,libssh.so.4,libsodium.so.23,libssl.so.3,libcrypto.so.3,libgssapi_krb5.so.2,libkrb5.so.3,libk5crypto.so.3,libkrb5support.so.0,libkeyutils.so.1,libbrotlidec.so.1,libbrotlicommon.so.1,libidn2.so.0} $THIRDPARTY_LIB/ - cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/ + cp /usr/local/lib/libgflags.so.2.3 $THIRDPARTY_LIB/ find /usr/ -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null } From c09bf8022b4679e730625b5651e5af14b130d2ee Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 28 Jul 2026 05:09:47 +0100 Subject: [PATCH 20/20] fix mac build Signed-off-by: Yuan --- .github/workflows/velox_weekly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index ec93d85455b..af8a17530c8 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -330,7 +330,7 @@ jobs: distribution: temurin java-version: 17 - name: Install Dependencies - run: brew install wget pkgconfig abseil ccache + run: brew install wget pkgconfig ccache - name: Get Ccache from Apache Stash uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 with: