[VL] Followup on #12321 to add a basic tpc test for weekly job

.github/workflows/velox_weekly.yml

@@ -25,9 +25,10 @@ on:
 
 env:
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-  TPCH_TEST: "env GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare 
+  TPCH_TEST: "env GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare
             --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1"
   INSTALL_PREFIX: /usr/local
+  CCACHE_DIR: "${{ github.workspace }}/.ccache"
 
 
 concurrency:
@@ -47,6 +48,30 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Install Stash restore dependencies # the stash action needs gh, jq, and python3 (>=3.7), absent from this container
+        run: |
+          if [ "${{ matrix.os }}" = "centos:8" ]; then
+            sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
+            sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
+            # centos:8 ships python3.6 by default; install python39 from AppStream and override the symlink
+            dnf install -y python39 && ln -sf /usr/bin/python3.9 /usr/bin/python3
+          else
+            dnf install -y python3
+          fi
+          case "$(uname -m)" in
+            x86_64) ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+          esac
+          curl -fsSL "https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz" \
+            | tar -xz -C /usr/local --strip-components=1 "gh_2.63.2_linux_${ARCH}/bin/gh"
+          curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}"
+          chmod +x /usr/local/bin/jq
+          gh --version && jq --version
+      - name: Get Ccache from Apache Stash
+        uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-centos-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }}
       - name: build
         run: |
           if [ "${{ matrix.os }}" = "centos:8" ]; then
@@ -63,33 +88,74 @@ jobs:
           dnf install -y epel-release sudo dnf
           dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
           source /opt/rh/gcc-toolset-12/enable || exit 1
-          yum install -y java-1.8.0-openjdk-devel patch wget git perl
+          yum install -y java-1.8.0-openjdk-devel patch wget git perl ccache
           export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
           export PATH=$JAVA_HOME/bin:$PATH
+          export CCACHE_MAXSIZE=1G
+          mkdir -p $CCACHE_DIR
           export VELOX_BUILD_SHARED=ON
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
+          cd $GITHUB_WORKSPACE/tools/gluten-it
+          $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
+      - name: Clean Ccache tmp before save
+        if: always()
+        run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+      - name: Save Ccache to Apache Stash
+        if: always()
+        uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-centos-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }}
+          include-hidden-files: true
+          retention-days: '7'
+      - name: Run TPC-H
+        run: |
+          export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+          echo "JAVA_HOME: $JAVA_HOME"
+          cd $GITHUB_WORKSPACE/tools/gluten-it
+          $TPCH_TEST
 
   weekly-build-on-ubuntu:
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu:22.04" ]
+        os: [ "ubuntu:22.04", "ubuntu:24.04" ]
     if: ${{ startsWith(github.repository, 'apache/') }}
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+      - name: Install Stash restore dependencies # the stash action needs gh, jq, and python3, absent from this container
+        run: |
+          case "$(uname -m)" in
+            x86_64) ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+          esac
+          apt-get update -y && apt-get install -y curl python3
+          curl -fsSL "https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz" \
+            | tar -xz -C /usr/local --strip-components=1 "gh_2.63.2_linux_${ARCH}/bin/gh"
+          curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}"
+          chmod +x /usr/local/bin/jq
+          gh --version && jq --version
+      - name: Get Ccache from Apache Stash
+        uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-ubuntu-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }}
       - name: build
         run: |
           # sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list
           # To avoid the prompt for region selection during installing tzdata.
           export DEBIAN_FRONTEND=noninteractive
           export INSTALL_PREFIX=/usr/
-          apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential cmake openjdk-8-jdk python3-pip
-          pip3 install cmake==3.31.4
+          apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential openjdk-8-jdk python3-pip ccache
           cd /tmp
-          wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz
-          tar -xzf gflags.tar.gz && cd gflags-2.2.2
+          wget https://github.com/Kitware/CMake/releases/download/v3.31.12/cmake-3.31.12-linux-$(uname -m).sh; \
+          chmod +x cmake-3.31.12-linux-$(uname -m).sh; \
+          ./cmake-3.31.12-linux-$(uname -m).sh --skip-license --prefix=/usr/local; \
+          rm -f cmake-3.31.12-linux-$(uname -m).sh;
+          wget https://github.com/gflags/gflags/archive/v2.3.0.tar.gz -O gflags.tar.gz
+          tar -xzf gflags.tar.gz && cd gflags-2.3.0
           cmake -B build \
             -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
             -DBUILD_SHARED_LIBS=ON \
@@ -109,8 +175,30 @@ jobs:
           cmake . && make install -j
           apt autoremove -y
           export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+          export CCACHE_MAXSIZE=1G
+          mkdir -p $CCACHE_DIR
           export VELOX_BUILD_SHARED=ON
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
+          cd $GITHUB_WORKSPACE/tools/gluten-it
+          $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
+      - name: Clean Ccache tmp before save
+        if: always()
+        run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+      - name: Save Ccache to Apache Stash
+        if: always()
+        uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-ubuntu-${{ matrix.os }}-${{ hashFiles('ep/build-velox/src/**') }}
+          include-hidden-files: true
+          retention-days: '7'
+      - name: Run TPC-H
+        run: |
+          export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+          echo "JAVA_HOME: $JAVA_HOME"
+          cd $GITHUB_WORKSPACE/tools/gluten-it
+          $TPCH_TEST
+
 
   weekly-build-on-openeuler:
     strategy:
@@ -124,16 +212,45 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl
+          dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl ccache curl python3
           echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
+      - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container
+        run: |
+          case "$(uname -m)" in
+            x86_64) ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+          esac
+          curl -fsSL "https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz" \
+            | tar -xz -C /usr/local --strip-components=1 "gh_2.63.2_linux_${ARCH}/bin/gh"
+          curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}"
+          chmod +x /usr/local/bin/jq
+          gh --version && jq --version
+      - name: Get Ccache from Apache Stash
+        uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-openeuler-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }}
       - name: Build
         run: |
           echo "JAVA_HOME: $JAVA_HOME"
+          export CCACHE_MAXSIZE=1G
+          mkdir -p $CCACHE_DIR
           cd $GITHUB_WORKSPACE/
           ./dev/package.sh --spark_version=3.5
           cd $GITHUB_WORKSPACE/tools/gluten-it
           $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
           cp $GITHUB_WORKSPACE/package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar package/target/lib/
+      - name: Clean Ccache tmp before save
+        if: always()
+        run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+      - name: Save Ccache to Apache Stash
+        if: always()
+        uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-openeuler-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }}
+          include-hidden-files: true
+          retention-days: '7'
       - name: Run TPC-H
         run: |
           echo "JAVA_HOME: $JAVA_HOME"
@@ -153,16 +270,45 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl
+          dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch git perl curl python3
           echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
+      - name: Install Stash restore dependencies # the stash action needs gh and jq, absent from this container
+        run: |
+          case "$(uname -m)" in
+            x86_64) ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+          esac
+          curl -fsSL "https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz" \
+            | tar -xz -C /usr/local --strip-components=1 "gh_2.63.2_linux_${ARCH}/bin/gh"
+          curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}"
+          chmod +x /usr/local/bin/jq
+          gh --version && jq --version
+      - name: Get Ccache from Apache Stash
+        uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-openeuler-vcpkg-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }}
       - name: Build
         run: |
           echo "JAVA_HOME: $JAVA_HOME"
+          export CCACHE_MAXSIZE=1G
+          mkdir -p $CCACHE_DIR
           cd $GITHUB_WORKSPACE/
           ./dev/vcpkg/setup-build-depends.sh
           ./dev/package-vcpkg.sh --build_arrow=ON --spark_version=3.5
           cd $GITHUB_WORKSPACE/tools/gluten-it
           $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
+      - name: Clean Ccache tmp before save
+        if: always()
+        run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+      - name: Save Ccache to Apache Stash
+        if: always()
+        uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-openeuler-vcpkg-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }}
+          include-hidden-files: true
+          retention-days: '7'
       - name: Run TPC-H
         run: |
           echo "JAVA_HOME: $JAVA_HOME"
@@ -184,6 +330,25 @@ jobs:
           distribution: temurin
           java-version: 17
       - name: Install Dependencies
-        run: brew install wget pkgconfig abseil ccache
+        run: brew install wget pkgconfig ccache
+      - name: Get Ccache from Apache Stash
+        uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-macos-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }}
       - name: Build
-        run: ./dev/buildbundle-veloxbe.sh --run_setup_script=ON --build_arrow=ON --build_tests=ON --build_benchmarks=ON --spark_version=3.5
+        run: |
+          export CCACHE_MAXSIZE=1G
+          mkdir -p $CCACHE_DIR
+          ./dev/buildbundle-veloxbe.sh --run_setup_script=ON --build_arrow=ON --build_tests=ON --build_benchmarks=ON --spark_version=3.5
+      - name: Clean Ccache tmp before save
+        if: always()
+        run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+      - name: Save Ccache to Apache Stash
+        if: always()
+        uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-weekly-macos-${{runner.arch}}-${{ hashFiles('ep/build-velox/src/**') }}
+          include-hidden-files: true
+          retention-days: '7'

backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala

@@ -38,7 +38,7 @@ class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
     loader.loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so")
     loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so")
-    loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+    loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
     loader.loadAndCreateLink("libglog.so.0", "libglog.so")
     loader.loadAndCreateLink("libre2.so.10", "libre2.so")
     loader.loadAndCreateLink("libzstd.so.1", "libzstd.so")

backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala

@@ -42,7 +42,7 @@ class SharedLibraryLoaderCentos8 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
     loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so")
     loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so")
-    loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+    loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
     loader.loadAndCreateLink("libglog.so.1", "libglog.so")
     loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so")
     loader.loadAndCreateLink("libre2.so.0", "libre2.so")

backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala

@@ -38,7 +38,7 @@ class SharedLibraryLoaderCentos9 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
     loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so")
     loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
-    loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+    loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
     loader.loadAndCreateLink("libglog.so.1", "libglog.so")
     loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so")
     loader.loadAndCreateLink("libre2.so.9", "libre2.so")

backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala

@@ -33,7 +33,7 @@ class SharedLibraryLoaderDebian11 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so")
     loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so")
     loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
-    loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+    loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
     loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
     loader.loadAndCreateLink("librtmp.so.1", "librtmp.so")
     loader.loadAndCreateLink("libssh2.so.1", "libssh2.so")

backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala

@@ -39,7 +39,7 @@ class SharedLibraryLoaderDebian12 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so")
     loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so")
     loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so")
-    loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+    loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
     loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
     loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
     loader.loadAndCreateLink("librtmp.so.1", "librtmp.so")

backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala

@@ -36,7 +36,7 @@ class SharedLibraryLoaderOpenEuler2403 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
     loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so")
     loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
-    loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+    loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
     loader.loadAndCreateLink("libglog.so.1", "libglog.so")
     loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so")
     loader.loadAndCreateLink("libidn.so.12", "libidn.so")

backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala

@@ -38,7 +38,7 @@ class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
     loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so")
     loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
-    loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+    loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
     loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
     loader.loadAndCreateLink("libglog.so.0", "libglog.so")
     loader.loadAndCreateLink("libidn.so.12", "libidn.so")