[GLUTEN-7139][VL] Manage Velox Arrow through vcpkg

.github/workflows/velox_backend_arm.yml

@@ -144,6 +144,7 @@ jobs:
             --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1
 
   cpp-test-udf-test:
+    if: ${{ false }}
     runs-on: ubuntu-24.04-arm
     container: apache/gluten:centos-9-jdk8
     steps:

.github/workflows/velox_backend_x86.yml

@@ -985,6 +985,7 @@ jobs:
             **/gluten-ut/**/core.*
 
   cpp-test-udf-test:
+    if: ${{ false }}
     runs-on: ubuntu-22.04
     container: apache/gluten:centos-9-jdk8
     steps:
@@ -1068,6 +1069,7 @@ jobs:
           path: "**/target/*.log"
 
   build-cudf-centos-9:
+    if: ${{ false }}
     runs-on: ubuntu-22.04
     steps:
       - name: "node-cleanup" # by default the free runner does not have enough disk space
@@ -1118,6 +1120,7 @@ jobs:
           "
 
   build-fast-build-test:
+    if: ${{ false }}
     runs-on: ubuntu-22.04
     container: apache/gluten:centos-8-jdk17
     steps:

.github/workflows/velox_weekly.yml

@@ -41,7 +41,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "centos:8", "quay.io/centos/centos:stream9" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+    if: ${{ false }}
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
     steps:
@@ -74,7 +74,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "ubuntu:22.04" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+    if: ${{ false }}
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
     steps:
@@ -117,7 +117,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "openeuler/openeuler:24.03" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+    if: ${{ false }}
     runs-on: ubuntu-24.04-arm
     container: ${{ matrix.os }}
     steps:
@@ -146,7 +146,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "openeuler/openeuler:24.03" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+    if: ${{ false }}
     runs-on: ubuntu-24.04-arm
     container: ${{ matrix.os }}
     steps:
@@ -160,7 +160,7 @@ jobs:
           echo "JAVA_HOME: $JAVA_HOME"
           cd $GITHUB_WORKSPACE/
           ./dev/vcpkg/setup-build-depends.sh
-          ./dev/package-vcpkg.sh --build_arrow=ON --spark_version=3.5
+          ./dev/package-vcpkg.sh --spark_version=3.5
           cd $GITHUB_WORKSPACE/tools/gluten-it
           $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
       - name: Run TPC-H
@@ -172,7 +172,7 @@ jobs:
   weekly-build-on-macos:
     strategy:
       fail-fast: false
-    if: ${{ startsWith(github.repository, 'apache/') }}
+    if: ${{ false }}
     runs-on: macos-15
     env:
       INSTALL_PREFIX: /tmp/deps-install

cpp/CMake/ConfigArrow.cmake

@@ -20,6 +20,14 @@ set(ARROW_STATIC_LIBRARY_SUFFIX ".a")
 set(ARROW_LIB_NAME "arrow")
 set(ARROW_BUNDLED_DEPS "arrow_bundled_dependencies")
 
+if(ENABLE_GLUTEN_VCPKG)
+  find_package(Arrow CONFIG REQUIRED)
+  if(NOT TARGET Arrow::arrow)
+    add_library(Arrow::arrow INTERFACE IMPORTED)
+    target_link_libraries(Arrow::arrow INTERFACE Arrow::arrow_static)
+  endif()
+endif()
+
 set(ARROW_INSTALL_DIR "${ARROW_HOME}/install")
 set(ARROW_LIB_DIR "${ARROW_INSTALL_DIR}/lib")
 set(ARROW_LIB64_DIR "${ARROW_INSTALL_DIR}/lib64")

cpp/CMakeLists.txt

@@ -68,6 +68,7 @@ endif()
 
 set(GLUTEN_PREFIX_INCLUDE_DIRS)
 if(CMAKE_INSTALL_PREFIX
+   AND NOT ENABLE_GLUTEN_VCPKG
    AND NOT CMAKE_INSTALL_PREFIX STREQUAL "/usr/local"
    AND IS_DIRECTORY "${CMAKE_INSTALL_PREFIX}/include")
   list(APPEND GLUTEN_PREFIX_INCLUDE_DIRS "${CMAKE_INSTALL_PREFIX}/include")
@@ -191,6 +192,11 @@ find_package(Threads REQUIRED)
 find_package(JNI REQUIRED)
 
 find_package(glog REQUIRED)
+if(ENABLE_GLUTEN_VCPKG
+   AND TARGET glog::glog
+   AND NOT TARGET google::glog)
+  add_library(google::glog ALIAS glog::glog)
+endif()
 
 if(BUILD_TESTS)
   set(GLUTEN_GTEST_MIN_VERSION "1.13.0")

cpp/core/CMakeLists.txt

@@ -201,8 +201,10 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
   endif()
 endif()
 
-find_arrow_lib(${ARROW_LIB_NAME})
-find_arrow_lib(${ARROW_BUNDLED_DEPS})
+if(NOT ENABLE_GLUTEN_VCPKG)
+  find_arrow_lib(${ARROW_LIB_NAME})
+  find_arrow_lib(${ARROW_BUNDLED_DEPS})
+endif()
 
 if(ENABLE_QAT)
   include(BuildQATzip)
@@ -251,8 +253,10 @@ if(BUILD_TESTS)
   add_subdirectory(tests)
 endif()
 
-target_link_libraries(gluten PUBLIC Arrow::arrow
-                                    Arrow::arrow_bundled_dependencies)
+target_link_libraries(gluten PUBLIC Arrow::arrow)
+if(NOT ENABLE_GLUTEN_VCPKG)
+  target_link_libraries(gluten PUBLIC Arrow::arrow_bundled_dependencies)
+endif()
 target_link_libraries(gluten PRIVATE google::glog)
 
 install(TARGETS gluten DESTINATION ${CMAKE_INSTALL_LIBDIR})

cpp/velox/CMakeLists.txt

@@ -246,6 +246,11 @@ endif()
 
 add_library(velox SHARED ${VELOX_SRCS})
 
+if(ENABLE_GLUTEN_VCPKG)
+  find_package(xxHash CONFIG REQUIRED)
+  target_link_libraries(velox PUBLIC xxHash::xxhash)
+endif()
+
 # Keep ABI sensitive Velox feature macros in sync
 set(GLUTEN_VELOX_CMAKE_CACHE "${VELOX_BUILD_PATH}/CMakeCache.txt")
 set(GLUTEN_VELOX_ENABLE_PARQUET ${VELOX_ENABLE_PARQUET})
@@ -342,12 +347,28 @@ if(BUILD_TESTS)
   import_library(
     facebook::velox::file_test_utils
     ${VELOX_BUILD_PATH}/velox/common/file/tests/libvelox_file_test_utils.a)
+  import_library(
+    facebook::velox::vector_fuzzer_util
+    ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer_util.a)
+  import_library(
+    facebook::velox::constrained_vector_generator
+    ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_constrained_vector_generator.a
+  )
+  target_link_libraries(facebook::velox::constrained_vector_generator
+                        INTERFACE facebook::velox::vector_fuzzer_util)
+  import_library(
+    facebook::velox::vector_fuzzer
+    ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer.a)
+  target_link_libraries(
+    facebook::velox::vector_fuzzer
+    INTERFACE facebook::velox::vector_fuzzer_util
+              facebook::velox::constrained_vector_generator)
   import_library(
     facebook::velox::exec_test_lib
     ${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_exec_test_lib.a)
   target_link_libraries(
     facebook::velox::exec_test_lib
-    INTERFACE facebook::velox::vector_test_lib
+    INTERFACE facebook::velox::vector_test_lib facebook::velox::vector_fuzzer
               facebook::velox::dwio_common_test
               facebook::velox::file_test_utils)
   target_link_libraries(velox PUBLIC facebook::velox::exec_test_lib)
@@ -422,8 +443,10 @@ endif()
 target_link_libraries(velox PUBLIC Folly::folly)
 
 # Link thrift libraries - check vcpkg first, then fall back to system libraries
-# Determine vcpkg triplet directory based on architecture
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
+# Use the active vcpkg triplet when available, then fall back to architecture.
+if(VCPKG_TARGET_TRIPLET)
+  set(VCPKG_TRIPLET_DIR "${VCPKG_TARGET_TRIPLET}")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
   set(VCPKG_TRIPLET_DIR "x64-linux-avx")
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
   set(VCPKG_TRIPLET_DIR "arm64-linux-neon")
@@ -443,8 +466,8 @@ if(DEFINED VCPKG_INSTALLED_DIR
   )
   target_link_libraries(
     velox
-    PRIVATE ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a
-            ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a)
+    PUBLIC ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a
+           ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a)
 else()
   set(THRIFT_LIBRARY_DIRS)
   if(CMAKE_INSTALL_PREFIX)

cpp/velox/benchmarks/CMakeLists.txt

@@ -20,6 +20,9 @@ target_include_directories(
                                 ${CMAKE_SOURCE_DIR}/core)
 target_link_libraries(velox_benchmark_common PUBLIC velox benchmark::benchmark
                                                     google::glog)
+if(BUILD_TESTS)
+  target_link_libraries(velox_benchmark_common PRIVATE GTest::gtest)
+endif()
 
 function(add_velox_benchmark BM_EXEC BM_FILE)
   add_executable(${BM_EXEC} ${BM_FILE})

cpp/velox/operators/writer/VeloxParquetDataSource.h

@@ -17,7 +17,6 @@
 
 #pragma once
 
-#include <arrow/filesystem/filesystem.h>
 #include <arrow/memory_pool.h>
 #include <arrow/record_batch.h>
 #include <arrow/type.h>

dev/builddep-veloxbe-inc.sh

@@ -139,9 +139,11 @@ else
         export VCPKG_TRIPLET="x64-linux-avx"
     fi
 
+    export VCPKG_MANIFEST_DIR="$GLUTEN_DIR/dev/vcpkg"
     export VCPKG_TRIPLET_INSTALL_DIR="$GLUTEN_DIR/dev/vcpkg/vcpkg_installed/${VCPKG_TRIPLET}"
     export CMAKE_TOOLCHAIN_FILE="$GLUTEN_DIR/dev/vcpkg/toolchain.cmake"
-    export PKG_CONFIG_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
+    unset PKG_CONFIG_PATH
+    export PKG_CONFIG_LIBDIR="${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig"
     export GLUTEN_VCPKG_ENABLED="${VCPKG_ROOT}"
 fi
 

dev/builddeps-veloxbe.sh

@@ -45,6 +45,7 @@ VELOX_BRANCH=""
 VELOX_HOME="$GLUTEN_DIR/ep/build-velox/build/velox_ep"
 VELOX_PARAMETER=""
 BUILD_ARROW=ON
+BUILD_ARROW_EXPLICIT=OFF
 SPARK_VERSION=ALL
 
 # set default number of threads as cpu cores minus 2
@@ -141,6 +142,7 @@ do
         ;;
         --build_arrow=*)
         BUILD_ARROW="${arg#*=}"
+        BUILD_ARROW_EXPLICIT=ON
         shift # Remove argument name from processing
         ;;
         --num_threads=*)
@@ -158,6 +160,18 @@ do
     esac
 done
 
+function vcpkg_is_active {
+    [ "$ENABLE_VCPKG" = "ON" ] || [ -n "${GLUTEN_VCPKG_ENABLED:-}" ]
+}
+
+if vcpkg_is_active; then
+    if [ "$BUILD_ARROW_EXPLICIT" = "ON" ] && [ "$BUILD_ARROW" = "ON" ]; then
+        echo "ERROR: --build_arrow=ON is deprecated with --enable_vcpkg=ON; Arrow is managed by Gluten vcpkg." >&2
+        exit 1
+    fi
+    BUILD_ARROW=OFF
+fi
+
 if [[ "$(uname)" == "Darwin" ]]; then
     export INSTALL_PREFIX=${INSTALL_PREFIX:-${VELOX_HOME}/deps-install}
     if [[ "$INSTALL_PREFIX" == "/usr/local" || "$INSTALL_PREFIX" == /usr/local/* ]]; then
@@ -196,12 +210,43 @@ function concat_velox_param {
     VELOX_PARAMETER+="--run_setup_script=$RUN_SETUP_SCRIPT "
 }
 
+function print_vcpkg_diagnostics {
+    local phase="$1"
+    echo "::group::vcpkg diagnostics: ${phase}"
+    printf '%s\n' \
+        "PATH=${PATH}" \
+        "CC=${CC:-}" \
+        "CXX=${CXX:-}" \
+        "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}" \
+        "LIBRARY_PATH=${LIBRARY_PATH:-}" \
+        "COMPILER_PATH=${COMPILER_PATH:-}" \
+        "GCC_EXEC_PREFIX=${GCC_EXEC_PREFIX:-}" \
+        "PKG_CONFIG_PATH=${PKG_CONFIG_PATH:-}" \
+        "PKG_CONFIG_LIBDIR=${PKG_CONFIG_LIBDIR:-}" \
+        "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-}" \
+        "CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-}"
+    for tool in gcc g++ cc c++ ld flex; do
+        echo "--- ${tool}"
+        type -a "${tool}" || true
+        command -v "${tool}" | xargs -r readlink -f || true
+    done
+    for compiler in gcc g++ cc c++; do
+        echo "--- ${compiler} details"
+        "${compiler}" --version 2>/dev/null | head -n 1 || true
+        "${compiler}" -print-file-name=libstdc++.a 2>/dev/null || true
+        "${compiler}" -print-search-dirs 2>/dev/null || true
+    done
+    find /usr/include /usr/local/include /opt/rh -name FlexLexer.h -print 2>/dev/null || true
+    echo "::endgroup::"
+}
 
 if [ "$ENABLE_VCPKG" = "ON" ]; then
     # vcpkg will install static depends and init build environment
     BUILD_OPTIONS="--build_tests=$BUILD_TESTS --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS \
                    --enable_hdfs=$ENABLE_HDFS --enable_abfs=$ENABLE_ABFS"
+    print_vcpkg_diagnostics "before env.sh"
     source ./dev/vcpkg/env.sh ${BUILD_OPTIONS}
+    print_vcpkg_diagnostics "after env.sh"
 fi
 
 # Supported Spark versions
@@ -223,6 +268,10 @@ concat_velox_param
 export VELOX_HOME
 
 function build_arrow {
+  if vcpkg_is_active; then
+    echo "ERROR: build_arrow is deprecated with --enable_vcpkg=ON; Arrow is managed by Gluten vcpkg." >&2
+    return 1
+  fi
   local GLUTEN_BUILD_TYPE="$BUILD_TYPE"
   if [ ! -d "$VELOX_HOME" ]; then
     get_velox
@@ -273,8 +322,10 @@ function build_gluten_cpp {
   )
 
   if [ -n "${INSTALL_PREFIX:-}" ]; then
-    GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX")
     GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX")
+    if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ]; then
+      GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX")
+    fi
   fi
   if [ $OS == 'Darwin' ]; then
     if [[ -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then

dev/ci-velox-buildstatic-centos-7-enhanced-features.sh

@@ -17,6 +17,12 @@
 
 set -e
 
+rm -f /usr/local/bin/flex /usr/local/include/FlexLexer.h
+yum install -y flex
+hash -r
+test "$(flex --version)" = "flex 2.5.37"
+test -f /usr/include/FlexLexer.h
+
 export NUM_THREADS=2
 cp ep/build-velox/src/socket.h /usr/include/asm-generic/socket.h 
 ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=ON --build_benchmarks=ON \

dev/ci-velox-buildstatic-centos-7.sh

@@ -17,6 +17,12 @@
 
 set -e
 
+rm -f /usr/local/bin/flex /usr/local/include/FlexLexer.h
+yum install -y flex
+hash -r
+test "$(flex --version)" = "flex 2.5.37"
+test -f /usr/include/FlexLexer.h
+
 export NUM_THREADS=4
 cp ep/build-velox/src/socket.h /usr/include/asm-generic/socket.h 
 ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \

dev/docker/Dockerfile.centos7-gcc13-static-build

@@ -43,10 +43,10 @@ RUN set -ex; \
     yum remove gcc -y && yum clean all; \
     yes | cp -rf /usr/share/aclocal/* /usr/local/share/aclocal/; \
     mkdir -p ${VCPKG_PATH}; \
-    echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \
+    echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \
     cd /opt/gluten; \
-    bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \
-                                       --enable_hdfs=ON --enable_abfs=ON  build_arrow; \
+    bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \
+                                       --enable_hdfs=ON --enable_abfs=ON'; \
     ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \
     dnf clean all; \
     rm -rf /opt/gluten; \