From 916536a9171d7d573fcaf90eaa0b74391c657f3f Mon Sep 17 00:00:00 2001 From: Chang chen Date: Sat, 25 Jul 2026 09:49:21 +0800 Subject: [PATCH 1/9] [VL] Manage Velox Arrow through vcpkg Clean vcpkg builds skipped Gluten's standalone Arrow installer but could still fall back to bundled or host Arrow, mixing Arrow versions and losing static dependency usage requirements. The Arrow testing library also exposed incompatible Boost.Process selection with newer Boost releases. Add an Arrow 18 overlay with testing support, align Boost at 1.84, force vcpkg-managed SYSTEM resolution, and consume exported CMake targets with the required static dependency propagation. Isolate host package paths, deprecate duplicate vcpkg Arrow installation, and update callers so clean builds cannot silently depend on /usr/local. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81cd27bb-c736-4cf5-b65f-2aeb9f954996 --- .github/workflows/velox_weekly.yml | 2 +- cpp/CMake/ConfigArrow.cmake | 8 + cpp/CMakeLists.txt | 6 + cpp/core/CMakeLists.txt | 12 +- cpp/velox/CMakeLists.txt | 27 +++- cpp/velox/benchmarks/CMakeLists.txt | 3 + .../operators/writer/VeloxParquetDataSource.h | 1 - dev/builddep-veloxbe-inc.sh | 4 +- dev/builddeps-veloxbe.sh | 22 ++- .../Dockerfile.centos7-gcc13-static-build | 4 +- dev/docker/Dockerfile.centos7-static-build | 4 +- .../Dockerfile.centos8-gcc13-static-build | 4 +- dev/docker/Dockerfile.centos8-static-build | 4 +- dev/docker/Dockerfile.centos9-static-build | 4 +- dev/vcpkg/env.sh | 5 +- dev/vcpkg/ports/arrow/android.patch | 27 ++++ .../ports/arrow/arrow-testing-boost.patch | 64 ++++++++ .../arrow/arrow-testing-static-boost.patch | 38 +++++ dev/vcpkg/ports/arrow/msvc-static-name.patch | 13 ++ dev/vcpkg/ports/arrow/portfile.cmake | 152 ++++++++++++++++++ dev/vcpkg/ports/arrow/thrift.patch | 22 +++ dev/vcpkg/ports/arrow/usage | 4 + dev/vcpkg/ports/arrow/usage-acero | 3 + dev/vcpkg/ports/arrow/usage-dataset | 3 + dev/vcpkg/ports/arrow/usage-flight | 3 + dev/vcpkg/ports/arrow/usage-flightsql | 3 + dev/vcpkg/ports/arrow/usage-parquet | 3 + dev/vcpkg/ports/arrow/utf8proc.patch | 13 ++ dev/vcpkg/ports/arrow/vcpkg.json | 149 +++++++++++++++++ dev/vcpkg/toolchain.cmake | 32 +++- dev/vcpkg/vcpkg-configuration.json | 6 + dev/vcpkg/vcpkg.json | 5 + ep/build-velox/src/build-velox.sh | 23 ++- ep/build-velox/src/get-velox.sh | 6 +- .../workload/benchmark_velox/initialize.ipynb | 12 -- 35 files changed, 642 insertions(+), 49 deletions(-) create mode 100644 dev/vcpkg/ports/arrow/android.patch create mode 100644 dev/vcpkg/ports/arrow/arrow-testing-boost.patch create mode 100644 dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch create mode 100644 dev/vcpkg/ports/arrow/msvc-static-name.patch create mode 100644 dev/vcpkg/ports/arrow/portfile.cmake create mode 100644 dev/vcpkg/ports/arrow/thrift.patch create mode 100644 dev/vcpkg/ports/arrow/usage create mode 100644 dev/vcpkg/ports/arrow/usage-acero create mode 100644 dev/vcpkg/ports/arrow/usage-dataset create mode 100644 dev/vcpkg/ports/arrow/usage-flight create mode 100644 dev/vcpkg/ports/arrow/usage-flightsql create mode 100644 dev/vcpkg/ports/arrow/usage-parquet create mode 100644 dev/vcpkg/ports/arrow/utf8proc.patch create mode 100644 dev/vcpkg/ports/arrow/vcpkg.json diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 43bac7fc6b7..561831b819b 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -160,7 +160,7 @@ jobs: echo "JAVA_HOME: $JAVA_HOME" cd $GITHUB_WORKSPACE/ ./dev/vcpkg/setup-build-depends.sh - ./dev/package-vcpkg.sh --build_arrow=ON --spark_version=3.5 + ./dev/package-vcpkg.sh --spark_version=3.5 cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 - name: Run TPC-H diff --git a/cpp/CMake/ConfigArrow.cmake b/cpp/CMake/ConfigArrow.cmake index 0d604a91e69..4cf061a7692 100644 --- a/cpp/CMake/ConfigArrow.cmake +++ b/cpp/CMake/ConfigArrow.cmake @@ -20,6 +20,14 @@ set(ARROW_STATIC_LIBRARY_SUFFIX ".a") set(ARROW_LIB_NAME "arrow") set(ARROW_BUNDLED_DEPS "arrow_bundled_dependencies") +if(ENABLE_GLUTEN_VCPKG) + find_package(Arrow CONFIG REQUIRED) + if(NOT TARGET Arrow::arrow) + add_library(Arrow::arrow INTERFACE IMPORTED) + target_link_libraries(Arrow::arrow INTERFACE Arrow::arrow_static) + endif() +endif() + set(ARROW_INSTALL_DIR "${ARROW_HOME}/install") set(ARROW_LIB_DIR "${ARROW_INSTALL_DIR}/lib") set(ARROW_LIB64_DIR "${ARROW_INSTALL_DIR}/lib64") diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cce42a9bbe5..369aa864177 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -68,6 +68,7 @@ endif() set(GLUTEN_PREFIX_INCLUDE_DIRS) if(CMAKE_INSTALL_PREFIX + AND NOT ENABLE_GLUTEN_VCPKG AND NOT CMAKE_INSTALL_PREFIX STREQUAL "/usr/local" AND IS_DIRECTORY "${CMAKE_INSTALL_PREFIX}/include") list(APPEND GLUTEN_PREFIX_INCLUDE_DIRS "${CMAKE_INSTALL_PREFIX}/include") @@ -191,6 +192,11 @@ find_package(Threads REQUIRED) find_package(JNI REQUIRED) find_package(glog REQUIRED) +if(ENABLE_GLUTEN_VCPKG + AND TARGET glog::glog + AND NOT TARGET google::glog) + add_library(google::glog ALIAS glog::glog) +endif() if(BUILD_TESTS) set(GLUTEN_GTEST_MIN_VERSION "1.13.0") diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index 240567b8ff1..77c4b692e4d 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -201,8 +201,10 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) endif() endif() -find_arrow_lib(${ARROW_LIB_NAME}) -find_arrow_lib(${ARROW_BUNDLED_DEPS}) +if(NOT ENABLE_GLUTEN_VCPKG) + find_arrow_lib(${ARROW_LIB_NAME}) + find_arrow_lib(${ARROW_BUNDLED_DEPS}) +endif() if(ENABLE_QAT) include(BuildQATzip) @@ -251,8 +253,10 @@ if(BUILD_TESTS) add_subdirectory(tests) endif() -target_link_libraries(gluten PUBLIC Arrow::arrow - Arrow::arrow_bundled_dependencies) +target_link_libraries(gluten PUBLIC Arrow::arrow) +if(NOT ENABLE_GLUTEN_VCPKG) + target_link_libraries(gluten PUBLIC Arrow::arrow_bundled_dependencies) +endif() target_link_libraries(gluten PRIVATE google::glog) install(TARGETS gluten DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 78baafef8f4..d324479109f 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -246,6 +246,11 @@ endif() add_library(velox SHARED ${VELOX_SRCS}) +if(ENABLE_GLUTEN_VCPKG) + find_package(xxHash CONFIG REQUIRED) + target_link_libraries(velox PUBLIC xxHash::xxhash) +endif() + # Keep ABI sensitive Velox feature macros in sync set(GLUTEN_VELOX_CMAKE_CACHE "${VELOX_BUILD_PATH}/CMakeCache.txt") set(GLUTEN_VELOX_ENABLE_PARQUET ${VELOX_ENABLE_PARQUET}) @@ -342,12 +347,28 @@ if(BUILD_TESTS) import_library( facebook::velox::file_test_utils ${VELOX_BUILD_PATH}/velox/common/file/tests/libvelox_file_test_utils.a) + import_library( + facebook::velox::vector_fuzzer_util + ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer_util.a) + import_library( + facebook::velox::constrained_vector_generator + ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_constrained_vector_generator.a + ) + target_link_libraries(facebook::velox::constrained_vector_generator + INTERFACE facebook::velox::vector_fuzzer_util) + import_library( + facebook::velox::vector_fuzzer + ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer.a) + target_link_libraries( + facebook::velox::vector_fuzzer + INTERFACE facebook::velox::vector_fuzzer_util + facebook::velox::constrained_vector_generator) import_library( facebook::velox::exec_test_lib ${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_exec_test_lib.a) target_link_libraries( facebook::velox::exec_test_lib - INTERFACE facebook::velox::vector_test_lib + INTERFACE facebook::velox::vector_test_lib facebook::velox::vector_fuzzer facebook::velox::dwio_common_test facebook::velox::file_test_utils) target_link_libraries(velox PUBLIC facebook::velox::exec_test_lib) @@ -443,8 +464,8 @@ if(DEFINED VCPKG_INSTALLED_DIR ) target_link_libraries( velox - PRIVATE ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a - ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a) + PUBLIC ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a + ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a) else() set(THRIFT_LIBRARY_DIRS) if(CMAKE_INSTALL_PREFIX) diff --git a/cpp/velox/benchmarks/CMakeLists.txt b/cpp/velox/benchmarks/CMakeLists.txt index 617ada61717..474818e6043 100644 --- a/cpp/velox/benchmarks/CMakeLists.txt +++ b/cpp/velox/benchmarks/CMakeLists.txt @@ -20,6 +20,9 @@ target_include_directories( ${CMAKE_SOURCE_DIR}/core) target_link_libraries(velox_benchmark_common PUBLIC velox benchmark::benchmark google::glog) +if(BUILD_TESTS) + target_link_libraries(velox_benchmark_common PRIVATE GTest::gtest) +endif() function(add_velox_benchmark BM_EXEC BM_FILE) add_executable(${BM_EXEC} ${BM_FILE}) diff --git a/cpp/velox/operators/writer/VeloxParquetDataSource.h b/cpp/velox/operators/writer/VeloxParquetDataSource.h index dac97168d5a..cf51df1d5d7 100644 --- a/cpp/velox/operators/writer/VeloxParquetDataSource.h +++ b/cpp/velox/operators/writer/VeloxParquetDataSource.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include #include diff --git a/dev/builddep-veloxbe-inc.sh b/dev/builddep-veloxbe-inc.sh index 9869690ca9a..50610993424 100755 --- a/dev/builddep-veloxbe-inc.sh +++ b/dev/builddep-veloxbe-inc.sh @@ -139,9 +139,11 @@ else export VCPKG_TRIPLET="x64-linux-avx" fi + export VCPKG_MANIFEST_DIR="$GLUTEN_DIR/dev/vcpkg" export VCPKG_TRIPLET_INSTALL_DIR="$GLUTEN_DIR/dev/vcpkg/vcpkg_installed/${VCPKG_TRIPLET}" export CMAKE_TOOLCHAIN_FILE="$GLUTEN_DIR/dev/vcpkg/toolchain.cmake" - export PKG_CONFIG_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH:-}" + unset PKG_CONFIG_PATH + export PKG_CONFIG_LIBDIR="${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig" export GLUTEN_VCPKG_ENABLED="${VCPKG_ROOT}" fi diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index f36be37377d..0e7b3a89bb2 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -45,6 +45,7 @@ VELOX_BRANCH="" VELOX_HOME="$GLUTEN_DIR/ep/build-velox/build/velox_ep" VELOX_PARAMETER="" BUILD_ARROW=ON +BUILD_ARROW_EXPLICIT=OFF SPARK_VERSION=ALL # set default number of threads as cpu cores minus 2 @@ -141,6 +142,7 @@ do ;; --build_arrow=*) BUILD_ARROW="${arg#*=}" + BUILD_ARROW_EXPLICIT=ON shift # Remove argument name from processing ;; --num_threads=*) @@ -158,6 +160,18 @@ do esac done +function vcpkg_is_active { + [ "$ENABLE_VCPKG" = "ON" ] || [ -n "${GLUTEN_VCPKG_ENABLED:-}" ] +} + +if vcpkg_is_active; then + if [ "$BUILD_ARROW_EXPLICIT" = "ON" ] && [ "$BUILD_ARROW" = "ON" ]; then + echo "ERROR: --build_arrow=ON is deprecated with --enable_vcpkg=ON; Arrow is managed by Gluten vcpkg." >&2 + exit 1 + fi + BUILD_ARROW=OFF +fi + if [[ "$(uname)" == "Darwin" ]]; then export INSTALL_PREFIX=${INSTALL_PREFIX:-${VELOX_HOME}/deps-install} if [[ "$INSTALL_PREFIX" == "/usr/local" || "$INSTALL_PREFIX" == /usr/local/* ]]; then @@ -223,6 +237,10 @@ concat_velox_param export VELOX_HOME function build_arrow { + if vcpkg_is_active; then + echo "ERROR: build_arrow is deprecated with --enable_vcpkg=ON; Arrow is managed by Gluten vcpkg." >&2 + return 1 + fi local GLUTEN_BUILD_TYPE="$BUILD_TYPE" if [ ! -d "$VELOX_HOME" ]; then get_velox @@ -273,8 +291,10 @@ function build_gluten_cpp { ) if [ -n "${INSTALL_PREFIX:-}" ]; then - GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX") GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX") + if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ]; then + GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX") + fi fi if [ $OS == 'Darwin' ]; then if [[ -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then diff --git a/dev/docker/Dockerfile.centos7-gcc13-static-build b/dev/docker/Dockerfile.centos7-gcc13-static-build index c9e6ede10e6..928188e4396 100644 --- a/dev/docker/Dockerfile.centos7-gcc13-static-build +++ b/dev/docker/Dockerfile.centos7-gcc13-static-build @@ -43,10 +43,10 @@ RUN set -ex; \ yum remove gcc -y && yum clean all; \ yes | cp -rf /usr/share/aclocal/* /usr/local/share/aclocal/; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ cd /opt/gluten; \ bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + --enable_hdfs=ON --enable_abfs=ON true; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ dnf clean all; \ rm -rf /opt/gluten; \ diff --git a/dev/docker/Dockerfile.centos7-static-build b/dev/docker/Dockerfile.centos7-static-build index 638f144e0b5..f213cb1eb67 100644 --- a/dev/docker/Dockerfile.centos7-static-build +++ b/dev/docker/Dockerfile.centos7-static-build @@ -39,12 +39,12 @@ RUN set -ex; \ echo "check_certificate = off" >> ~/.wgetrc; \ cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ cd /opt/gluten; \ source /opt/rh/devtoolset-11/enable; \ source /opt/rh/rh-git227/enable; \ bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + --enable_hdfs=ON --enable_abfs=ON true; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ dnf clean all; \ rm -rf /opt/gluten; \ diff --git a/dev/docker/Dockerfile.centos8-gcc13-static-build b/dev/docker/Dockerfile.centos8-gcc13-static-build index 37e609f64a1..06fa804d893 100644 --- a/dev/docker/Dockerfile.centos8-gcc13-static-build +++ b/dev/docker/Dockerfile.centos8-gcc13-static-build @@ -37,14 +37,14 @@ RUN set -ex; \ dnf clean all; \ git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ if [ "$(uname -m)" = "aarch64" ]; then \ export CPU_TARGET="aarch64"; \ export VCPKG_FORCE_SYSTEM_BINARIES=1; \ fi; \ cd /opt/gluten; \ bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + --enable_hdfs=ON --enable_abfs=ON true; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ rm -rf /opt/gluten; \ rm -rf /root/.cache/ccache; diff --git a/dev/docker/Dockerfile.centos8-static-build b/dev/docker/Dockerfile.centos8-static-build index d6e2d5ec1c8..dc5f81cb18c 100644 --- a/dev/docker/Dockerfile.centos8-static-build +++ b/dev/docker/Dockerfile.centos8-static-build @@ -37,7 +37,7 @@ RUN set -ex; \ git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \ cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ if [ "$(uname -m)" = "aarch64" ]; then \ export CPU_TARGET="aarch64"; \ export VCPKG_FORCE_SYSTEM_BINARIES=1; \ @@ -46,7 +46,7 @@ RUN set -ex; \ cd /opt/gluten; \ export SSL_VERIFY=false; \ bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + --enable_hdfs=ON --enable_abfs=ON true; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ rm -rf /opt/gluten; \ rm -rf /root/.cache/ccache; diff --git a/dev/docker/Dockerfile.centos9-static-build b/dev/docker/Dockerfile.centos9-static-build index c83c99a465a..751b7c05c6a 100644 --- a/dev/docker/Dockerfile.centos9-static-build +++ b/dev/docker/Dockerfile.centos9-static-build @@ -33,7 +33,7 @@ RUN set -ex; \ git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \ cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ if [ "$(uname -m)" = "aarch64" ]; then \ export CPU_TARGET="aarch64"; \ export VCPKG_FORCE_SYSTEM_BINARIES=1; \ @@ -41,7 +41,7 @@ RUN set -ex; \ source /opt/rh/gcc-toolset-12/enable; \ cd /opt/gluten; \ bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + --enable_hdfs=ON --enable_abfs=ON true; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ rm -rf /opt/gluten; \ rm -rf /root/.cache/ccache; diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh index a86a953e2b5..066183b7477 100755 --- a/dev/vcpkg/env.sh +++ b/dev/vcpkg/env.sh @@ -31,6 +31,9 @@ export VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET} ${SCRIPT_ROOT}/init.sh "$@" +unset PKG_CONFIG_PATH +export PKG_CONFIG_LIBDIR=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig + if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then EXPORT_TOOLS_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/tools/protobuf" # The scripts depends on environment $CMAKE_TOOLCHAIN_FILE, which requires @@ -45,9 +48,7 @@ if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then export VCPKG_ROOT=${VCPKG_ROOT} export VCPKG_MANIFEST_DIR=${SCRIPT_ROOT} export VCPKG_TRIPLET=${VCPKG_TRIPLET} - export CMAKE_TOOLCHAIN_FILE=${SCRIPT_ROOT}/toolchain.cmake - export PKG_CONFIG_PATH=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig:${PKG_CONFIG_PATH:-} export PATH="${EXPORT_TOOLS_PATH}:$PATH" export GLUTEN_VCPKG_ENABLED=${VCPKG_ROOT} diff --git a/dev/vcpkg/ports/arrow/android.patch b/dev/vcpkg/ports/arrow/android.patch new file mode 100644 index 00000000000..4eb412d1a52 --- /dev/null +++ b/dev/vcpkg/ports/arrow/android.patch @@ -0,0 +1,27 @@ +diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt +index 6dc8358..2b91efa 100644 +--- a/cpp/src/arrow/CMakeLists.txt ++++ b/cpp/src/arrow/CMakeLists.txt +@@ -166,7 +166,7 @@ if(WIN32) + list(APPEND ARROW_SYSTEM_LINK_LIBS "ws2_32") + endif() + +-if(NOT WIN32 AND NOT APPLE) ++if(NOT WIN32 AND NOT APPLE AND NOT ANDROID) + # Pass -lrt on Linux only + list(APPEND ARROW_SYSTEM_LINK_LIBS rt) + endif() +diff --git a/cpp/src/arrow/vendored/musl/strptime.c b/cpp/src/arrow/vendored/musl/strptime.c +index 41912fd..0ea36e9 100644 +--- a/cpp/src/arrow/vendored/musl/strptime.c ++++ b/cpp/src/arrow/vendored/musl/strptime.c +@@ -18,7 +18,9 @@ + #undef HAVE_LANGINFO + + #ifndef _WIN32 ++# if !(defined(__ANDROID__) && __ANDROID_API__ < 26) + #define HAVE_LANGINFO 1 ++# endif + #endif + + #ifdef HAVE_LANGINFO diff --git a/dev/vcpkg/ports/arrow/arrow-testing-boost.patch b/dev/vcpkg/ports/arrow/arrow-testing-boost.patch new file mode 100644 index 00000000000..a3ef9267565 --- /dev/null +++ b/dev/vcpkg/ports/arrow/arrow-testing-boost.patch @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This patch replaces Boost::process (header-only, no CMake target) with +# Boost::filesystem and Boost::system (the actual libraries it would have linked) +# Based on https://github.com/apache/arrow/pull/45424 + +diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +index db151b4..1da99f1 100644 +--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -1266,13 +1266,19 @@ endif() + # - Gandiva has a compile-time (header-only) dependency on Boost, not runtime. + # - Tests need Boost at runtime. + # - S3FS and Flight benchmarks need Boost at runtime. ++# - arrow_testing uses boost::filesystem. So arrow_testing requires ++# Boost library. (boost::filesystem isn't header-only.) But if we ++# use arrow_testing as a static library without ++# using arrow::util::Process, we don't need boost::filesystem. + if(ARROW_BUILD_INTEGRATION + OR ARROW_BUILD_TESTS + OR (ARROW_FLIGHT AND (ARROW_TESTING OR ARROW_BUILD_BENCHMARKS)) +- OR (ARROW_S3 AND ARROW_BUILD_BENCHMARKS)) ++ OR (ARROW_S3 AND ARROW_BUILD_BENCHMARKS) ++ OR (ARROW_TESTING AND ARROW_BUILD_SHARED)) + set(ARROW_USE_BOOST TRUE) + set(ARROW_BOOST_REQUIRE_LIBRARY TRUE) + elseif(ARROW_GANDIVA ++ OR ARROW_TESTING + OR ARROW_WITH_THRIFT + OR (NOT ARROW_USE_NATIVE_INT128)) + set(ARROW_USE_BOOST TRUE) +diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt +index c911f0f..84673d4 100644 +--- a/cpp/src/arrow/CMakeLists.txt ++++ b/cpp/src/arrow/CMakeLists.txt +@@ -645,11 +645,11 @@ else() + endif() + + set(ARROW_TESTING_SHARED_LINK_LIBS arrow_shared ${ARROW_GTEST_GTEST}) +-set(ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::flatbuffers RapidJSON Boost::process) ++set(ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::flatbuffers RapidJSON Boost::filesystem Boost::system) + set(ARROW_TESTING_STATIC_LINK_LIBS + arrow::flatbuffers + RapidJSON +- Boost::process ++ Boost::filesystem Boost::system + arrow_static + ${ARROW_GTEST_GTEST}) + set(ARROW_TESTING_SHARED_INSTALL_INTERFACE_LIBS Arrow::arrow_shared) diff --git a/dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch b/dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch new file mode 100644 index 00000000000..c87382699cb --- /dev/null +++ b/dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -1267,15 +1267,12 @@ endif() + # - Gandiva has a compile-time (header-only) dependency on Boost, not runtime. + # - Tests need Boost at runtime. + # - S3FS and Flight benchmarks need Boost at runtime. +-# - arrow_testing uses boost::filesystem. So arrow_testing requires +-# Boost library. (boost::filesystem isn't header-only.) But if we +-# use arrow_testing as a static library without +-# using arrow::util::Process, we don't need boost::filesystem. ++# - arrow_testing directly links Boost::filesystem and Boost::system. + if(ARROW_BUILD_INTEGRATION + OR ARROW_BUILD_TESTS + OR (ARROW_FLIGHT AND (ARROW_TESTING OR ARROW_BUILD_BENCHMARKS)) + OR (ARROW_S3 AND ARROW_BUILD_BENCHMARKS) +- OR (ARROW_TESTING AND ARROW_BUILD_SHARED)) ++ OR ARROW_TESTING) + set(ARROW_USE_BOOST TRUE) + set(ARROW_BOOST_REQUIRE_LIBRARY TRUE) + elseif(ARROW_GANDIVA diff --git a/dev/vcpkg/ports/arrow/msvc-static-name.patch b/dev/vcpkg/ports/arrow/msvc-static-name.patch new file mode 100644 index 00000000000..61d99ebac53 --- /dev/null +++ b/dev/vcpkg/ports/arrow/msvc-static-name.patch @@ -0,0 +1,13 @@ +diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake +index 391c43e0a..50f6d3d3c 100644 +--- a/cpp/cmake_modules/BuildUtils.cmake ++++ b/cpp/cmake_modules/BuildUtils.cmake +@@ -427,7 +427,7 @@ function(ADD_ARROW_LIB LIB_NAME) + target_include_directories(${LIB_NAME}_static PRIVATE ${ARG_PRIVATE_INCLUDES}) + endif() + +- if(MSVC_TOOLCHAIN) ++ if(MSVC_TOOLCHAIN AND 0) + set(LIB_NAME_STATIC ${LIB_NAME}_static) + else() + set(LIB_NAME_STATIC ${LIB_NAME}) diff --git a/dev/vcpkg/ports/arrow/portfile.cmake b/dev/vcpkg/ports/arrow/portfile.cmake new file mode 100644 index 00000000000..6942dbd46dc --- /dev/null +++ b/dev/vcpkg/ports/arrow/portfile.cmake @@ -0,0 +1,152 @@ +vcpkg_download_distfile( + ARCHIVE_PATH + URLS "https://archive.apache.org/dist/arrow/arrow-${VERSION}/apache-arrow-${VERSION}.tar.gz" + FILENAME apache-arrow-${VERSION}.tar.gz + SHA512 4df30ab5561da695eaa864422626b9898555d86ca56835c3b8a8ca93a1dbaf081582bb36e2440d1daf7e1dd48c76941f1152a4f25ce0dbcc1c2abe244a00c05e +) +vcpkg_extract_source_archive( + SOURCE_PATH + ARCHIVE ${ARCHIVE_PATH} + PATCHES + android.patch + msvc-static-name.patch + utf8proc.patch + thrift.patch + arrow-testing-boost.patch + arrow-testing-static-boost.patch +) + +vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + acero ARROW_ACERO + compute ARROW_COMPUTE + csv ARROW_CSV + cuda ARROW_CUDA + dataset ARROW_DATASET + filesystem ARROW_FILESYSTEM + flight ARROW_FLIGHT + flightsql ARROW_FLIGHT_SQL + gcs ARROW_GCS + jemalloc ARROW_JEMALLOC + json ARROW_JSON + mimalloc ARROW_MIMALLOC + orc ARROW_ORC + parquet ARROW_PARQUET + parquet PARQUET_REQUIRE_ENCRYPTION + s3 ARROW_S3 + testing ARROW_TESTING +) + +if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) + list(APPEND FEATURE_OPTIONS "-DARROW_USE_NATIVE_INT128=OFF") +endif() + +string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "dynamic" ARROW_BUILD_SHARED) +string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "static" ARROW_BUILD_STATIC) +string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "dynamic" ARROW_DEPENDENCY_USE_SHARED) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}/cpp" + OPTIONS + ${FEATURE_OPTIONS} + -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED} + -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC} + -DARROW_BUILD_TESTS=OFF + -DARROW_DEPENDENCY_SOURCE=SYSTEM + -DARROW_DEPENDENCY_USE_SHARED=${ARROW_DEPENDENCY_USE_SHARED} + -DARROW_PACKAGE_KIND=vcpkg + -DARROW_WITH_BROTLI=ON + -DARROW_WITH_BZ2=ON + -DARROW_WITH_LZ4=ON + -DARROW_WITH_SNAPPY=ON + -DARROW_WITH_ZLIB=ON + -DARROW_WITH_ZSTD=ON + -DBUILD_WARNING_LEVEL=PRODUCTION + -DZSTD_MSVC_LIB_PREFIX= + MAYBE_UNUSED_VARIABLES + ZSTD_MSVC_LIB_PREFIX +) + +vcpkg_cmake_install() +vcpkg_copy_pdbs() + +vcpkg_fixup_pkgconfig() + +if(EXISTS "${CURRENT_PACKAGES_DIR}/lib/arrow_static.lib") + message(FATAL_ERROR "Installed lib file should be named 'arrow.lib' via patching the upstream build.") +endif() + +if("dataset" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME arrowdataset + CONFIG_PATH lib/cmake/ArrowDataset + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() + +if("acero" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME arrowacero + CONFIG_PATH lib/cmake/ArrowAcero + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() + +if("flight" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME ArrowFlight + CONFIG_PATH lib/cmake/ArrowFlight + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() + +if("flightsql" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME ArrowFlightSql + CONFIG_PATH lib/cmake/ArrowFlightSql + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() + +if("parquet" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME parquet + CONFIG_PATH lib/cmake/Parquet + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() +vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/Arrow) + +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +if("parquet" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-parquet" usage-parquet) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-parquet}") +endif() +if("dataset" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-dataset" usage-dataset) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-dataset}") +endif() +if("acero" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-acero" usage-acero) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-acero}") +endif() + +if("flight" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-flight" usage-flight) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-flight}") +endif() + +if("flightsql" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-flightsql" usage-flightsql) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-flightsql}") +endif() + +if("example" IN_LIST FEATURES) + file(INSTALL "${SOURCE_PATH}/cpp/examples/minimal_build/" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}/example") +endif() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share/doc") + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE.txt") diff --git a/dev/vcpkg/ports/arrow/thrift.patch b/dev/vcpkg/ports/arrow/thrift.patch new file mode 100644 index 00000000000..ae284d65b85 --- /dev/null +++ b/dev/vcpkg/ports/arrow/thrift.patch @@ -0,0 +1,22 @@ +diff --git a/cpp/cmake_modules/FindThriftAlt.cmake b/cpp/cmake_modules/FindThriftAlt.cmake +index f3e4902..65ceac8 100644 +--- a/cpp/cmake_modules/FindThriftAlt.cmake ++++ b/cpp/cmake_modules/FindThriftAlt.cmake +@@ -45,7 +45,7 @@ endif() + # * https://github.com/apache/thrift/pull/2725 + # * https://github.com/apache/thrift/pull/2726 + # * https://github.com/conda-forge/thrift-cpp-feedstock/issues/68 +-if(NOT WIN32) ++ + set(find_package_args "") + if(ThriftAlt_FIND_VERSION) + list(APPEND find_package_args ${ThriftAlt_FIND_VERSION}) +@@ -61,7 +61,7 @@ if(NOT WIN32) + "${THRIFT_COMPILER}") + return() + endif() +-endif() ++ + + function(extract_thrift_version) + if(ThriftAlt_INCLUDE_DIR) diff --git a/dev/vcpkg/ports/arrow/usage b/dev/vcpkg/ports/arrow/usage new file mode 100644 index 00000000000..b07f1c9c32b --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage @@ -0,0 +1,4 @@ +The package arrow provides CMake targets: + + find_package(Arrow CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,Arrow::arrow_static,Arrow::arrow_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-acero b/dev/vcpkg/ports/arrow/usage-acero new file mode 100644 index 00000000000..05c4776b797 --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-acero @@ -0,0 +1,3 @@ + + find_package(ArrowAcero CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,ArrowAcero::arrow_acero_static,ArrowAcero::arrow_acero_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-dataset b/dev/vcpkg/ports/arrow/usage-dataset new file mode 100644 index 00000000000..8bedfb83246 --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-dataset @@ -0,0 +1,3 @@ + + find_package(ArrowDataset CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,ArrowDataset::arrow_dataset_static,ArrowDataset::arrow_dataset_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-flight b/dev/vcpkg/ports/arrow/usage-flight new file mode 100644 index 00000000000..c9656253f9e --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-flight @@ -0,0 +1,3 @@ + + find_package(ArrowFlight CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,ArrowFlight::arrow_flight_static,ArrowFlight::arrow_flight_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-flightsql b/dev/vcpkg/ports/arrow/usage-flightsql new file mode 100644 index 00000000000..3159c059036 --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-flightsql @@ -0,0 +1,3 @@ + + find_package(ArrowFlightSql CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,ArrowFlightSql::arrow_flight_sql_static,ArrowFlightSql::arrow_flight_sql_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-parquet b/dev/vcpkg/ports/arrow/usage-parquet new file mode 100644 index 00000000000..ce04039a64e --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-parquet @@ -0,0 +1,3 @@ + + find_package(Parquet CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,Parquet::parquet_static,Parquet::parquet_shared>") diff --git a/dev/vcpkg/ports/arrow/utf8proc.patch b/dev/vcpkg/ports/arrow/utf8proc.patch new file mode 100644 index 00000000000..eea9ca59e2e --- /dev/null +++ b/dev/vcpkg/ports/arrow/utf8proc.patch @@ -0,0 +1,13 @@ +diff --git a/cpp/cmake_modules/Findutf8proc.cmake b/cpp/cmake_modules/Findutf8proc.cmake +index e347414..83f2aa1 100644 +--- a/cpp/cmake_modules/Findutf8proc.cmake ++++ b/cpp/cmake_modules/Findutf8proc.cmake +@@ -30,7 +30,7 @@ if(ARROW_PACKAGE_KIND STREQUAL "vcpkg") + if(utf8proc_FIND_REQUIRED) + list(APPEND find_package_args REQUIRED) + endif() +- find_package(utf8proc NAMES unofficial-utf8proc ${find_package_args}) ++ find_package(utf8proc NAMES unofficial-utf8proc) + if(utf8proc_FOUND) + add_library(utf8proc::utf8proc ALIAS utf8proc) + return() diff --git a/dev/vcpkg/ports/arrow/vcpkg.json b/dev/vcpkg/ports/arrow/vcpkg.json new file mode 100644 index 00000000000..f84b79ce3fd --- /dev/null +++ b/dev/vcpkg/ports/arrow/vcpkg.json @@ -0,0 +1,149 @@ +{ + "name": "arrow", + "version": "18.0.0", + "description": "Cross-language development platform for in-memory analytics", + "homepage": "https://arrow.apache.org", + "license": "Apache-2.0", + "supports": "x64 | (arm64 & !windows)", + "dependencies": [ + "boost-filesystem", + "boost-multiprecision", + "boost-system", + "brotli", + "bzip2", + "gflags", + "lz4", + "openssl", + "re2", + "snappy", + "thrift", + "utf8proc", + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + }, + "xsimd", + "zlib", + "zstd" + ], + "default-features": [ + "csv", + "filesystem", + "json", + "parquet" + ], + "features": { + "acero": { + "description": "Acero support" + }, + "compute": { + "description": "Build all computational kernel functions" + }, + "csv": { + "description": "CSV support" + }, + "cuda": { + "description": "cuda support", + "dependencies": [ + "cuda" + ] + }, + "dataset": { + "description": "Dataset support" + }, + "example": { + "description": "Install the minimal example (source code)" + }, + "filesystem": { + "description": "Filesystem support" + }, + "flight": { + "description": "Arrow Flight RPC support", + "dependencies": [ + "abseil", + "c-ares", + "grpc", + "protobuf" + ] + }, + "flightsql": { + "description": "FlightSQL support", + "dependencies": [ + { + "name": "arrow", + "default-features": false, + "features": [ + "flight" + ] + } + ] + }, + "gcs": { + "description": "GCS support", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "storage" + ] + } + ] + }, + "jemalloc": { + "description": "jemalloc allocator", + "supports": "!windows" + }, + "json": { + "description": "JSON support", + "dependencies": [ + "rapidjson" + ] + }, + "mimalloc": { + "description": "mimalloc allocator", + "supports": "windows" + }, + "orc": { + "description": "ORC support", + "dependencies": [ + "orc" + ] + }, + "parquet": { + "description": "Parquet support", + "dependencies": [ + "rapidjson" + ] + }, + "s3": { + "description": "S3 support", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "cognito-identity", + "config", + "identity-management", + "s3", + "sts", + "transfer" + ] + } + ] + }, + "testing": { + "description": "Build the Arrow testing support library", + "dependencies": [ + "boost-process", + "gtest", + "rapidjson" + ] + } + } +} diff --git a/dev/vcpkg/toolchain.cmake b/dev/vcpkg/toolchain.cmake index f4db8e7af53..364433bd07f 100644 --- a/dev/vcpkg/toolchain.cmake +++ b/dev/vcpkg/toolchain.cmake @@ -33,11 +33,37 @@ set(VCPKG_HOST_TRIPLET $ENV{VCPKG_TRIPLET}) set(VCPKG_INSTALLED_DIR $ENV{VCPKG_MANIFEST_DIR}/vcpkg_installed) set(VCPKG_INSTALL_OPTIONS --no-print-usage) -# Force read CMAKE_PREFIX_PATH from env -set(CMAKE_PREFIX_PATH $ENV{CMAKE_PREFIX_PATH}) - +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) include($ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake) +if(DEFINED ENV{GLUTEN_VCPKG_PREFER_CONFIG} + AND NOT "$ENV{GLUTEN_VCPKG_PREFER_CONFIG}" STREQUAL "") + set(_GLUTEN_VCPKG_PREFER_CONFIG "$ENV{GLUTEN_VCPKG_PREFER_CONFIG}") +elseif(DEFINED CMAKE_FIND_PACKAGE_PREFER_CONFIG) + set(_GLUTEN_VCPKG_PREFER_CONFIG "${CMAKE_FIND_PACKAGE_PREFER_CONFIG}") +else() + set(_GLUTEN_VCPKG_PREFER_CONFIG ON) +endif() +set(CMAKE_FIND_PACKAGE_PREFER_CONFIG "${_GLUTEN_VCPKG_PREFER_CONFIG}" CACHE BOOL + "Prefer package configuration files." FORCE) +unset(_GLUTEN_VCPKG_PREFER_CONFIG) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY CACHE STRING "Search for packages only in root paths." FORCE) +set(CMAKE_FIND_USE_PACKAGE_REGISTRY OFF CACHE BOOL "Disable the user package registry." FORCE) +set(CMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY OFF CACHE BOOL "Disable the system package registry." FORCE) + +set(_GLUTEN_VCPKG_IGNORED_PREFIXES /usr/local) +if(APPLE) + list(APPEND _GLUTEN_VCPKG_IGNORED_PREFIXES /opt/homebrew /opt/local) +endif() +if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "") + list(APPEND _GLUTEN_VCPKG_IGNORED_PREFIXES "$ENV{CONDA_PREFIX}") +endif() +list(APPEND CMAKE_IGNORE_PREFIX_PATH ${_GLUTEN_VCPKG_IGNORED_PREFIXES}) +list(REMOVE_DUPLICATES CMAKE_IGNORE_PREFIX_PATH) +set(CMAKE_IGNORE_PREFIX_PATH "${CMAKE_IGNORE_PREFIX_PATH}" CACHE STRING + "Prefixes ignored by Gluten's vcpkg toolchain." FORCE) +unset(_GLUTEN_VCPKG_IGNORED_PREFIXES) + set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -static-libgcc") set(CMAKE_SHARED_LINKER_FLAGS "-static-libstdc++ -static-libgcc") diff --git a/dev/vcpkg/vcpkg-configuration.json b/dev/vcpkg/vcpkg-configuration.json index ba7c7f58b32..9728afa8304 100644 --- a/dev/vcpkg/vcpkg-configuration.json +++ b/dev/vcpkg/vcpkg-configuration.json @@ -3,6 +3,12 @@ "overlay-ports": [ "./ports" ], "overlay-triplets": [ "./triplets" ], "registries": [ + { + "kind": "git", + "repository": "https://github.com/Microsoft/vcpkg", + "baseline": "943c5ef1c8f6b5e6ced092b242c8299caae2ff01", + "packages": [ "boost", "boost-*"] + }, { "kind": "git", "repository": "https://github.com/Microsoft/vcpkg", diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json index 55cf7573644..8731204d193 100644 --- a/dev/vcpkg/vcpkg.json +++ b/dev/vcpkg/vcpkg.json @@ -12,6 +12,11 @@ "lz4", "double-conversion", "bzip2", + { + "name": "arrow", + "default-features": false, + "features": ["testing"] + }, "curl", "lzo", "re2", diff --git a/ep/build-velox/src/build-velox.sh b/ep/build-velox/src/build-velox.sh index 3e279bf6808..3301d2c7a63 100755 --- a/ep/build-velox/src/build-velox.sh +++ b/ep/build-velox/src/build-velox.sh @@ -114,7 +114,7 @@ function compile { -Wno-error=uninitialized -Wno-unknown-warning-option -Wno-deprecated-declarations' if [[ "$(uname)" == "Darwin" ]]; then CXX_FLAGS="$CXX_FLAGS -Wno-inconsistent-missing-override -Wno-macro-redefined" - if [[ -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then + if [[ -z "${GLUTEN_VCPKG_ENABLED:-}" && -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then # Add the dependency prefix as a system include: this finds deps that only # publish loose headers (e.g. xsimd) and demotes warnings in vendored # dependency headers (abseil's __is_trivially_relocatable, arrow's vendored @@ -128,7 +128,10 @@ function compile { -DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_RUNNER=OFF -DVELOX_SIMDJSON_SKIPUTF8VALIDATION=ON \ -DVELOX_ENABLE_GEO=OFF" if [ -n "${INSTALL_PREFIX:-}" ]; then - COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" + COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" + if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ]; then + COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX}" + fi fi if [[ "$(uname)" == "Darwin" && -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PREFIX_PATH=/usr/local" @@ -179,12 +182,20 @@ function compile { fi echo "NUM_THREADS_OPTS: $NUM_THREADS_OPTS" - export simdjson_SOURCE=AUTO - export Arrow_SOURCE=AUTO + local -a VELOX_MAKE_COMMAND=(make) + if [ -n "${GLUTEN_VCPKG_ENABLED:-}" ]; then + export VELOX_DEPENDENCY_SOURCE=SYSTEM + export simdjson_SOURCE=SYSTEM + export Arrow_SOURCE=SYSTEM + VELOX_MAKE_COMMAND=(env GLUTEN_VCPKG_PREFER_CONFIG=OFF make) + else + export simdjson_SOURCE=AUTO + export Arrow_SOURCE=AUTO + fi if [ $ARCH == 'x86_64' ]; then - make $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" + "${VELOX_MAKE_COMMAND[@]}" $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" elif [[ "$ARCH" == 'arm64' || "$ARCH" == 'aarch64' || "$ARCH" == "ppc64le" ]]; then - CPU_TARGET=$ARCH make $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" + CPU_TARGET=$ARCH "${VELOX_MAKE_COMMAND[@]}" $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" else echo "Unsupported arch: $ARCH" exit 1 diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 72865dc94a6..7f27e708d5a 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -153,11 +153,7 @@ function apply_provided_velox_patch { } function apply_compilation_fixes { - local SUDO_CMD="" - if [ "$OS" == "Linux" ]; then - SUDO_CMD="sudo" - fi - $SUDO_CMD cp ${CURRENT_DIR}/modify_arrow.patch ${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/ + cp ${CURRENT_DIR}/modify_arrow.patch ${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/ git add ${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/modify_arrow.patch # to avoid the file from being deleted by git clean -dffx :/ } diff --git a/tools/workload/benchmark_velox/initialize.ipynb b/tools/workload/benchmark_velox/initialize.ipynb index 1d3e6397b6b..be106993aab 100644 --- a/tools/workload/benchmark_velox/initialize.ipynb +++ b/tools/workload/benchmark_velox/initialize.ipynb @@ -2232,18 +2232,6 @@ " !git clone https://github.com/apache/gluten.git gluten" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "hidden": true - }, - "outputs": [], - "source": [ - "# Build Arrow for the first time build.\n", - "!sed -i 's/--build_arrow=OFF/--build_arrow=ON/' ~/gluten/dev/package-vcpkg.sh" - ] - }, { "cell_type": "code", "execution_count": null, From 2ab3f8d4ffc614246e74782f96c66ac4316dc1e0 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Sat, 25 Jul 2026 16:46:49 +0800 Subject: [PATCH 2/9] [VL] Use active vcpkg triplet for Thrift Use VCPKG_TARGET_TRIPLET when locating vcpkg FBThrift archives so custom triplets do not fall back to an architecture-derived directory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81cd27bb-c736-4cf5-b65f-2aeb9f954996 --- cpp/velox/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index d324479109f..abb7d09739b 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -443,8 +443,10 @@ endif() target_link_libraries(velox PUBLIC Folly::folly) # Link thrift libraries - check vcpkg first, then fall back to system libraries -# Determine vcpkg triplet directory based on architecture -if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") +# Use the active vcpkg triplet when available, then fall back to architecture. +if(VCPKG_TARGET_TRIPLET) + set(VCPKG_TRIPLET_DIR "${VCPKG_TARGET_TRIPLET}") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") set(VCPKG_TRIPLET_DIR "x64-linux-avx") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") set(VCPKG_TRIPLET_DIR "arm64-linux-neon") From 4b77376dea1733538e169c71bb6e301ab5ded608 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Sat, 25 Jul 2026 16:48:00 +0800 Subject: [PATCH 3/9] [VL] Populate vcpkg cache directly in static images Source the vcpkg environment explicitly in dependency-cache images instead of using a no-op positional command through builddeps-veloxbe.sh. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81cd27bb-c736-4cf5-b65f-2aeb9f954996 --- dev/docker/Dockerfile.centos7-gcc13-static-build | 4 ++-- dev/docker/Dockerfile.centos7-static-build | 4 ++-- dev/docker/Dockerfile.centos8-gcc13-static-build | 4 ++-- dev/docker/Dockerfile.centos8-static-build | 4 ++-- dev/docker/Dockerfile.centos9-static-build | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dev/docker/Dockerfile.centos7-gcc13-static-build b/dev/docker/Dockerfile.centos7-gcc13-static-build index 928188e4396..e351922af6d 100644 --- a/dev/docker/Dockerfile.centos7-gcc13-static-build +++ b/dev/docker/Dockerfile.centos7-gcc13-static-build @@ -45,8 +45,8 @@ RUN set -ex; \ mkdir -p ${VCPKG_PATH}; \ echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ cd /opt/gluten; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON true; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ dnf clean all; \ rm -rf /opt/gluten; \ diff --git a/dev/docker/Dockerfile.centos7-static-build b/dev/docker/Dockerfile.centos7-static-build index f213cb1eb67..fa3730035ca 100644 --- a/dev/docker/Dockerfile.centos7-static-build +++ b/dev/docker/Dockerfile.centos7-static-build @@ -43,8 +43,8 @@ RUN set -ex; \ cd /opt/gluten; \ source /opt/rh/devtoolset-11/enable; \ source /opt/rh/rh-git227/enable; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON true; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ dnf clean all; \ rm -rf /opt/gluten; \ diff --git a/dev/docker/Dockerfile.centos8-gcc13-static-build b/dev/docker/Dockerfile.centos8-gcc13-static-build index 06fa804d893..9ee5f874587 100644 --- a/dev/docker/Dockerfile.centos8-gcc13-static-build +++ b/dev/docker/Dockerfile.centos8-gcc13-static-build @@ -43,8 +43,8 @@ RUN set -ex; \ export VCPKG_FORCE_SYSTEM_BINARIES=1; \ fi; \ cd /opt/gluten; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON true; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ rm -rf /opt/gluten; \ rm -rf /root/.cache/ccache; diff --git a/dev/docker/Dockerfile.centos8-static-build b/dev/docker/Dockerfile.centos8-static-build index dc5f81cb18c..2f20ea64130 100644 --- a/dev/docker/Dockerfile.centos8-static-build +++ b/dev/docker/Dockerfile.centos8-static-build @@ -45,8 +45,8 @@ RUN set -ex; \ source /opt/rh/gcc-toolset-11/enable; \ cd /opt/gluten; \ export SSL_VERIFY=false; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON true; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ rm -rf /opt/gluten; \ rm -rf /root/.cache/ccache; diff --git a/dev/docker/Dockerfile.centos9-static-build b/dev/docker/Dockerfile.centos9-static-build index 751b7c05c6a..5dbd4319111 100644 --- a/dev/docker/Dockerfile.centos9-static-build +++ b/dev/docker/Dockerfile.centos9-static-build @@ -40,8 +40,8 @@ RUN set -ex; \ fi; \ source /opt/rh/gcc-toolset-12/enable; \ cd /opt/gluten; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON true; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ rm -rf /opt/gluten; \ rm -rf /root/.cache/ccache; From aa0b1a1f8d6589f3859c3aa632a859c76646ccb0 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Sat, 25 Jul 2026 21:20:39 +0800 Subject: [PATCH 4/9] [VL] Use Boost CMake packages with vcpkg Advance the scoped Boost 1.84 registry to its CMake packaging transition so Folly can resolve Boost config packages with CMP0167 enabled. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81cd27bb-c736-4cf5-b65f-2aeb9f954996 --- dev/vcpkg/vcpkg-configuration.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/vcpkg/vcpkg-configuration.json b/dev/vcpkg/vcpkg-configuration.json index 9728afa8304..a7aa80d84ec 100644 --- a/dev/vcpkg/vcpkg-configuration.json +++ b/dev/vcpkg/vcpkg-configuration.json @@ -6,7 +6,7 @@ { "kind": "git", "repository": "https://github.com/Microsoft/vcpkg", - "baseline": "943c5ef1c8f6b5e6ced092b242c8299caae2ff01", + "baseline": "bcf3d00d2116056fda0ce47615f6074ffecb7524", "packages": [ "boost", "boost-*"] }, { From cb24e4f5c55e32912d7642221e2fc67dbf32ba63 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Sat, 25 Jul 2026 21:20:40 +0800 Subject: [PATCH 5/9] [VL] Isolate pkg-config during vcpkg install Clear host pkg-config search paths before init.sh runs vcpkg install, preventing system and Conda packages from leaking into dependency builds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81cd27bb-c736-4cf5-b65f-2aeb9f954996 --- dev/vcpkg/env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh index 066183b7477..a1b749800d9 100755 --- a/dev/vcpkg/env.sh +++ b/dev/vcpkg/env.sh @@ -29,11 +29,11 @@ export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg" export VCPKG_TRIPLET=$([ "${CPU_TARGET:-}" = "aarch64" ] && echo "arm64-linux-neon" || echo "x64-linux-avx") export VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET} -${SCRIPT_ROOT}/init.sh "$@" - unset PKG_CONFIG_PATH export PKG_CONFIG_LIBDIR=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig +${SCRIPT_ROOT}/init.sh "$@" + if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then EXPORT_TOOLS_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/tools/protobuf" # The scripts depends on environment $CMAKE_TOOLCHAIN_FILE, which requires From 21e75727f42706c69531145e9c80a1b4f11c5a02 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Sun, 26 Jul 2026 00:15:13 +0800 Subject: [PATCH 6/9] [VL] Disable unused Arrow codecs in vcpkg Disable Brotli and BZ2 in the Arrow overlay to match Velox and avoid unresolved codec symbols when libarrow.a is consumed through Velox FindArrow targets. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81cd27bb-c736-4cf5-b65f-2aeb9f954996 --- dev/vcpkg/ports/arrow/portfile.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/vcpkg/ports/arrow/portfile.cmake b/dev/vcpkg/ports/arrow/portfile.cmake index 6942dbd46dc..dfa795c81c4 100644 --- a/dev/vcpkg/ports/arrow/portfile.cmake +++ b/dev/vcpkg/ports/arrow/portfile.cmake @@ -55,8 +55,8 @@ vcpkg_cmake_configure( -DARROW_DEPENDENCY_SOURCE=SYSTEM -DARROW_DEPENDENCY_USE_SHARED=${ARROW_DEPENDENCY_USE_SHARED} -DARROW_PACKAGE_KIND=vcpkg - -DARROW_WITH_BROTLI=ON - -DARROW_WITH_BZ2=ON + -DARROW_WITH_BROTLI=OFF + -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON From 67bde2d2098811bc6271dadae6cdc4a540331a84 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Sun, 26 Jul 2026 01:48:08 +0800 Subject: [PATCH 7/9] [VL] Add temporary vcpkg CI diagnostics Limit the diagnostic run to native vcpkg jobs and log compiler and Flex resolution before and after vcpkg environment setup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81cd27bb-c736-4cf5-b65f-2aeb9f954996 --- .github/workflows/velox_backend_arm.yml | 1 + .github/workflows/velox_backend_x86.yml | 3 +++ .github/workflows/velox_weekly.yml | 10 ++++---- dev/builddeps-veloxbe.sh | 31 +++++++++++++++++++++++++ dev/vcpkg/toolchain.cmake | 15 ++++++++++++ 5 files changed, 55 insertions(+), 5 deletions(-) diff --git a/.github/workflows/velox_backend_arm.yml b/.github/workflows/velox_backend_arm.yml index 2f7fa8fe036..deb24a79f4c 100644 --- a/.github/workflows/velox_backend_arm.yml +++ b/.github/workflows/velox_backend_arm.yml @@ -144,6 +144,7 @@ jobs: --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 cpp-test-udf-test: + if: ${{ false }} runs-on: ubuntu-24.04-arm container: apache/gluten:centos-9-jdk8 steps: diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 9b13dbd069a..06e76d8cee4 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -985,6 +985,7 @@ jobs: **/gluten-ut/**/core.* cpp-test-udf-test: + if: ${{ false }} runs-on: ubuntu-22.04 container: apache/gluten:centos-9-jdk8 steps: @@ -1068,6 +1069,7 @@ jobs: path: "**/target/*.log" build-cudf-centos-9: + if: ${{ false }} runs-on: ubuntu-22.04 steps: - name: "node-cleanup" # by default the free runner does not have enough disk space @@ -1118,6 +1120,7 @@ jobs: " build-fast-build-test: + if: ${{ false }} runs-on: ubuntu-22.04 container: apache/gluten:centos-8-jdk17 steps: diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 561831b819b..9493b0cc901 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -41,7 +41,7 @@ jobs: fail-fast: false matrix: os: [ "centos:8", "quay.io/centos/centos:stream9" ] - if: ${{ startsWith(github.repository, 'apache/') }} + if: ${{ false }} runs-on: ubuntu-22.04 container: ${{ matrix.os }} steps: @@ -74,7 +74,7 @@ jobs: fail-fast: false matrix: os: [ "ubuntu:22.04" ] - if: ${{ startsWith(github.repository, 'apache/') }} + if: ${{ false }} runs-on: ubuntu-22.04 container: ${{ matrix.os }} steps: @@ -117,7 +117,7 @@ jobs: fail-fast: false matrix: os: [ "openeuler/openeuler:24.03" ] - if: ${{ startsWith(github.repository, 'apache/') }} + if: ${{ false }} runs-on: ubuntu-24.04-arm container: ${{ matrix.os }} steps: @@ -146,7 +146,7 @@ jobs: fail-fast: false matrix: os: [ "openeuler/openeuler:24.03" ] - if: ${{ startsWith(github.repository, 'apache/') }} + if: ${{ false }} runs-on: ubuntu-24.04-arm container: ${{ matrix.os }} steps: @@ -172,7 +172,7 @@ jobs: weekly-build-on-macos: strategy: fail-fast: false - if: ${{ startsWith(github.repository, 'apache/') }} + if: ${{ false }} runs-on: macos-15 env: INSTALL_PREFIX: /tmp/deps-install diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 0e7b3a89bb2..0ebbf0c5735 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -210,12 +210,43 @@ function concat_velox_param { VELOX_PARAMETER+="--run_setup_script=$RUN_SETUP_SCRIPT " } +function print_vcpkg_diagnostics { + local phase="$1" + echo "::group::vcpkg diagnostics: ${phase}" + printf '%s\n' \ + "PATH=${PATH}" \ + "CC=${CC:-}" \ + "CXX=${CXX:-}" \ + "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}" \ + "LIBRARY_PATH=${LIBRARY_PATH:-}" \ + "COMPILER_PATH=${COMPILER_PATH:-}" \ + "GCC_EXEC_PREFIX=${GCC_EXEC_PREFIX:-}" \ + "PKG_CONFIG_PATH=${PKG_CONFIG_PATH:-}" \ + "PKG_CONFIG_LIBDIR=${PKG_CONFIG_LIBDIR:-}" \ + "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-}" \ + "CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-}" + for tool in gcc g++ cc c++ ld flex; do + echo "--- ${tool}" + type -a "${tool}" || true + command -v "${tool}" | xargs -r readlink -f || true + done + for compiler in gcc g++ cc c++; do + echo "--- ${compiler} details" + "${compiler}" --version 2>/dev/null | head -n 1 || true + "${compiler}" -print-file-name=libstdc++.a 2>/dev/null || true + "${compiler}" -print-search-dirs 2>/dev/null || true + done + find /usr/include /usr/local/include /opt/rh -name FlexLexer.h -print 2>/dev/null || true + echo "::endgroup::" +} if [ "$ENABLE_VCPKG" = "ON" ]; then # vcpkg will install static depends and init build environment BUILD_OPTIONS="--build_tests=$BUILD_TESTS --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS \ --enable_hdfs=$ENABLE_HDFS --enable_abfs=$ENABLE_ABFS" + print_vcpkg_diagnostics "before env.sh" source ./dev/vcpkg/env.sh ${BUILD_OPTIONS} + print_vcpkg_diagnostics "after env.sh" fi # Supported Spark versions diff --git a/dev/vcpkg/toolchain.cmake b/dev/vcpkg/toolchain.cmake index 364433bd07f..fc23df09142 100644 --- a/dev/vcpkg/toolchain.cmake +++ b/dev/vcpkg/toolchain.cmake @@ -64,6 +64,21 @@ set(CMAKE_IGNORE_PREFIX_PATH "${CMAKE_IGNORE_PREFIX_PATH}" CACHE STRING "Prefixes ignored by Gluten's vcpkg toolchain." FORCE) unset(_GLUTEN_VCPKG_IGNORED_PREFIXES) +find_program(_GLUTEN_DIAGNOSTIC_CXX NAMES c++ g++ NO_CACHE) +find_program(_GLUTEN_DIAGNOSTIC_FLEX NAMES flex NO_CACHE) +find_path(_GLUTEN_DIAGNOSTIC_FLEX_INCLUDE_DIR NAMES FlexLexer.h NO_CACHE) +message(STATUS "[Gluten vcpkg diagnostics] ENV{CC}=$ENV{CC}") +message(STATUS "[Gluten vcpkg diagnostics] ENV{CXX}=$ENV{CXX}") +message(STATUS "[Gluten vcpkg diagnostics] ENV{PATH}=$ENV{PATH}") +message(STATUS "[Gluten vcpkg diagnostics] CMAKE_C_COMPILER=${CMAKE_C_COMPILER}") +message(STATUS "[Gluten vcpkg diagnostics] CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") +message(STATUS "[Gluten vcpkg diagnostics] c++=${_GLUTEN_DIAGNOSTIC_CXX}") +message(STATUS "[Gluten vcpkg diagnostics] flex=${_GLUTEN_DIAGNOSTIC_FLEX}") +message(STATUS "[Gluten vcpkg diagnostics] FlexLexer.h=${_GLUTEN_DIAGNOSTIC_FLEX_INCLUDE_DIR}") +unset(_GLUTEN_DIAGNOSTIC_CXX) +unset(_GLUTEN_DIAGNOSTIC_FLEX) +unset(_GLUTEN_DIAGNOSTIC_FLEX_INCLUDE_DIR) + set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -static-libgcc") set(CMAKE_SHARED_LINKER_FLAGS "-static-libstdc++ -static-libgcc") From 731a85e7cbe2f2a68dc0901d68a5fcc7d5762041 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Sun, 26 Jul 2026 07:16:42 +0800 Subject: [PATCH 8/9] [VL] Test vcpkg CI root-cause fixes Pin the active compiler before vcpkg setup and test CentOS 7 repository Flex against the current Velox minimum requirement. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81cd27bb-c736-4cf5-b65f-2aeb9f954996 --- dev/ci-velox-buildstatic-centos-7-enhanced-features.sh | 6 ++++++ dev/ci-velox-buildstatic-centos-7.sh | 6 ++++++ dev/vcpkg/env.sh | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/dev/ci-velox-buildstatic-centos-7-enhanced-features.sh b/dev/ci-velox-buildstatic-centos-7-enhanced-features.sh index 63cb8ccb4a8..c7567599f81 100755 --- a/dev/ci-velox-buildstatic-centos-7-enhanced-features.sh +++ b/dev/ci-velox-buildstatic-centos-7-enhanced-features.sh @@ -17,6 +17,12 @@ set -e +yum install -y flex +mv /usr/local/bin/flex /usr/local/bin/flex-2.6.4 +hash -r +test "$(flex --version)" = "flex 2.5.37" +test -f /usr/include/FlexLexer.h + export NUM_THREADS=2 cp ep/build-velox/src/socket.h /usr/include/asm-generic/socket.h ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=ON --build_benchmarks=ON \ diff --git a/dev/ci-velox-buildstatic-centos-7.sh b/dev/ci-velox-buildstatic-centos-7.sh index 1990d4c7d38..c6f309e4f65 100755 --- a/dev/ci-velox-buildstatic-centos-7.sh +++ b/dev/ci-velox-buildstatic-centos-7.sh @@ -17,6 +17,12 @@ set -e +yum install -y flex +mv /usr/local/bin/flex /usr/local/bin/flex-2.6.4 +hash -r +test "$(flex --version)" = "flex 2.5.37" +test -f /usr/include/FlexLexer.h + export NUM_THREADS=4 cp ep/build-velox/src/socket.h /usr/include/asm-generic/socket.h ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh index a1b749800d9..c960733e71b 100755 --- a/dev/vcpkg/env.sh +++ b/dev/vcpkg/env.sh @@ -29,6 +29,14 @@ export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg" export VCPKG_TRIPLET=$([ "${CPU_TARGET:-}" = "aarch64" ] && echo "arm64-linux-neon" || echo "x64-linux-avx") export VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET} +if [ -z "${CC:-}" ]; then + CC="$(command -v cc)" +fi +if [ -z "${CXX:-}" ]; then + CXX="$(command -v c++)" +fi +export CC CXX + unset PKG_CONFIG_PATH export PKG_CONFIG_LIBDIR=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig From 07d3f887592f252bc2516d2ce96661c1052d738c Mon Sep 17 00:00:00 2001 From: Chang chen Date: Sun, 26 Jul 2026 08:23:45 +0800 Subject: [PATCH 9/9] [VL] Test reordered vcpkg isolation setup Apply package-only lookup after vcpkg initialization and test CentOS 7 repository Flex after removing the preinstalled source build. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81cd27bb-c736-4cf5-b65f-2aeb9f954996 --- dev/ci-velox-buildstatic-centos-7-enhanced-features.sh | 2 +- dev/ci-velox-buildstatic-centos-7.sh | 2 +- dev/vcpkg/env.sh | 8 -------- dev/vcpkg/toolchain.cmake | 1 - 4 files changed, 2 insertions(+), 11 deletions(-) diff --git a/dev/ci-velox-buildstatic-centos-7-enhanced-features.sh b/dev/ci-velox-buildstatic-centos-7-enhanced-features.sh index c7567599f81..a31a75eedca 100755 --- a/dev/ci-velox-buildstatic-centos-7-enhanced-features.sh +++ b/dev/ci-velox-buildstatic-centos-7-enhanced-features.sh @@ -17,8 +17,8 @@ set -e +rm -f /usr/local/bin/flex /usr/local/include/FlexLexer.h yum install -y flex -mv /usr/local/bin/flex /usr/local/bin/flex-2.6.4 hash -r test "$(flex --version)" = "flex 2.5.37" test -f /usr/include/FlexLexer.h diff --git a/dev/ci-velox-buildstatic-centos-7.sh b/dev/ci-velox-buildstatic-centos-7.sh index c6f309e4f65..9a84fc8b687 100755 --- a/dev/ci-velox-buildstatic-centos-7.sh +++ b/dev/ci-velox-buildstatic-centos-7.sh @@ -17,8 +17,8 @@ set -e +rm -f /usr/local/bin/flex /usr/local/include/FlexLexer.h yum install -y flex -mv /usr/local/bin/flex /usr/local/bin/flex-2.6.4 hash -r test "$(flex --version)" = "flex 2.5.37" test -f /usr/include/FlexLexer.h diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh index c960733e71b..a1b749800d9 100755 --- a/dev/vcpkg/env.sh +++ b/dev/vcpkg/env.sh @@ -29,14 +29,6 @@ export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg" export VCPKG_TRIPLET=$([ "${CPU_TARGET:-}" = "aarch64" ] && echo "arm64-linux-neon" || echo "x64-linux-avx") export VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET} -if [ -z "${CC:-}" ]; then - CC="$(command -v cc)" -fi -if [ -z "${CXX:-}" ]; then - CXX="$(command -v c++)" -fi -export CC CXX - unset PKG_CONFIG_PATH export PKG_CONFIG_LIBDIR=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig diff --git a/dev/vcpkg/toolchain.cmake b/dev/vcpkg/toolchain.cmake index fc23df09142..e2150a3ac06 100644 --- a/dev/vcpkg/toolchain.cmake +++ b/dev/vcpkg/toolchain.cmake @@ -33,7 +33,6 @@ set(VCPKG_HOST_TRIPLET $ENV{VCPKG_TRIPLET}) set(VCPKG_INSTALLED_DIR $ENV{VCPKG_MANIFEST_DIR}/vcpkg_installed) set(VCPKG_INSTALL_OPTIONS --no-print-usage) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) include($ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake) if(DEFINED ENV{GLUTEN_VCPKG_PREFER_CONFIG}