diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 43bac7fc6b7..561831b819b 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -160,7 +160,7 @@ jobs: echo "JAVA_HOME: $JAVA_HOME" cd $GITHUB_WORKSPACE/ ./dev/vcpkg/setup-build-depends.sh - ./dev/package-vcpkg.sh --build_arrow=ON --spark_version=3.5 + ./dev/package-vcpkg.sh --spark_version=3.5 cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5 - name: Run TPC-H diff --git a/cpp/CMake/ConfigArrow.cmake b/cpp/CMake/ConfigArrow.cmake index 0d604a91e69..4cf061a7692 100644 --- a/cpp/CMake/ConfigArrow.cmake +++ b/cpp/CMake/ConfigArrow.cmake @@ -20,6 +20,14 @@ set(ARROW_STATIC_LIBRARY_SUFFIX ".a") set(ARROW_LIB_NAME "arrow") set(ARROW_BUNDLED_DEPS "arrow_bundled_dependencies") +if(ENABLE_GLUTEN_VCPKG) + find_package(Arrow CONFIG REQUIRED) + if(NOT TARGET Arrow::arrow) + add_library(Arrow::arrow INTERFACE IMPORTED) + target_link_libraries(Arrow::arrow INTERFACE Arrow::arrow_static) + endif() +endif() + set(ARROW_INSTALL_DIR "${ARROW_HOME}/install") set(ARROW_LIB_DIR "${ARROW_INSTALL_DIR}/lib") set(ARROW_LIB64_DIR "${ARROW_INSTALL_DIR}/lib64") diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cce42a9bbe5..369aa864177 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -68,6 +68,7 @@ endif() set(GLUTEN_PREFIX_INCLUDE_DIRS) if(CMAKE_INSTALL_PREFIX + AND NOT ENABLE_GLUTEN_VCPKG AND NOT CMAKE_INSTALL_PREFIX STREQUAL "/usr/local" AND IS_DIRECTORY "${CMAKE_INSTALL_PREFIX}/include") list(APPEND GLUTEN_PREFIX_INCLUDE_DIRS "${CMAKE_INSTALL_PREFIX}/include") @@ -191,6 +192,11 @@ find_package(Threads REQUIRED) find_package(JNI REQUIRED) find_package(glog REQUIRED) +if(ENABLE_GLUTEN_VCPKG + AND TARGET glog::glog + AND NOT TARGET google::glog) + add_library(google::glog ALIAS glog::glog) +endif() if(BUILD_TESTS) set(GLUTEN_GTEST_MIN_VERSION "1.13.0") diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index 240567b8ff1..77c4b692e4d 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -201,8 +201,10 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) endif() endif() -find_arrow_lib(${ARROW_LIB_NAME}) -find_arrow_lib(${ARROW_BUNDLED_DEPS}) +if(NOT ENABLE_GLUTEN_VCPKG) + find_arrow_lib(${ARROW_LIB_NAME}) + find_arrow_lib(${ARROW_BUNDLED_DEPS}) +endif() if(ENABLE_QAT) include(BuildQATzip) @@ -251,8 +253,10 @@ if(BUILD_TESTS) add_subdirectory(tests) endif() -target_link_libraries(gluten PUBLIC Arrow::arrow - Arrow::arrow_bundled_dependencies) +target_link_libraries(gluten PUBLIC Arrow::arrow) +if(NOT ENABLE_GLUTEN_VCPKG) + target_link_libraries(gluten PUBLIC Arrow::arrow_bundled_dependencies) +endif() target_link_libraries(gluten PRIVATE google::glog) install(TARGETS gluten DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 78baafef8f4..abb7d09739b 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -246,6 +246,11 @@ endif() add_library(velox SHARED ${VELOX_SRCS}) +if(ENABLE_GLUTEN_VCPKG) + find_package(xxHash CONFIG REQUIRED) + target_link_libraries(velox PUBLIC xxHash::xxhash) +endif() + # Keep ABI sensitive Velox feature macros in sync set(GLUTEN_VELOX_CMAKE_CACHE "${VELOX_BUILD_PATH}/CMakeCache.txt") set(GLUTEN_VELOX_ENABLE_PARQUET ${VELOX_ENABLE_PARQUET}) @@ -342,12 +347,28 @@ if(BUILD_TESTS) import_library( facebook::velox::file_test_utils ${VELOX_BUILD_PATH}/velox/common/file/tests/libvelox_file_test_utils.a) + import_library( + facebook::velox::vector_fuzzer_util + ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer_util.a) + import_library( + facebook::velox::constrained_vector_generator + ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_constrained_vector_generator.a + ) + target_link_libraries(facebook::velox::constrained_vector_generator + INTERFACE facebook::velox::vector_fuzzer_util) + import_library( + facebook::velox::vector_fuzzer + ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer.a) + target_link_libraries( + facebook::velox::vector_fuzzer + INTERFACE facebook::velox::vector_fuzzer_util + facebook::velox::constrained_vector_generator) import_library( facebook::velox::exec_test_lib ${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_exec_test_lib.a) target_link_libraries( facebook::velox::exec_test_lib - INTERFACE facebook::velox::vector_test_lib + INTERFACE facebook::velox::vector_test_lib facebook::velox::vector_fuzzer facebook::velox::dwio_common_test facebook::velox::file_test_utils) target_link_libraries(velox PUBLIC facebook::velox::exec_test_lib) @@ -422,8 +443,10 @@ endif() target_link_libraries(velox PUBLIC Folly::folly) # Link thrift libraries - check vcpkg first, then fall back to system libraries -# Determine vcpkg triplet directory based on architecture -if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") +# Use the active vcpkg triplet when available, then fall back to architecture. +if(VCPKG_TARGET_TRIPLET) + set(VCPKG_TRIPLET_DIR "${VCPKG_TARGET_TRIPLET}") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") set(VCPKG_TRIPLET_DIR "x64-linux-avx") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") set(VCPKG_TRIPLET_DIR "arm64-linux-neon") @@ -443,8 +466,8 @@ if(DEFINED VCPKG_INSTALLED_DIR ) target_link_libraries( velox - PRIVATE ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a - ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a) + PUBLIC ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a + ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a) else() set(THRIFT_LIBRARY_DIRS) if(CMAKE_INSTALL_PREFIX) diff --git a/cpp/velox/benchmarks/CMakeLists.txt b/cpp/velox/benchmarks/CMakeLists.txt index 617ada61717..474818e6043 100644 --- a/cpp/velox/benchmarks/CMakeLists.txt +++ b/cpp/velox/benchmarks/CMakeLists.txt @@ -20,6 +20,9 @@ target_include_directories( ${CMAKE_SOURCE_DIR}/core) target_link_libraries(velox_benchmark_common PUBLIC velox benchmark::benchmark google::glog) +if(BUILD_TESTS) + target_link_libraries(velox_benchmark_common PRIVATE GTest::gtest) +endif() function(add_velox_benchmark BM_EXEC BM_FILE) add_executable(${BM_EXEC} ${BM_FILE}) diff --git a/cpp/velox/operators/writer/VeloxParquetDataSource.h b/cpp/velox/operators/writer/VeloxParquetDataSource.h index dac97168d5a..cf51df1d5d7 100644 --- a/cpp/velox/operators/writer/VeloxParquetDataSource.h +++ b/cpp/velox/operators/writer/VeloxParquetDataSource.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include #include diff --git a/dev/builddep-veloxbe-inc.sh b/dev/builddep-veloxbe-inc.sh index 9869690ca9a..50610993424 100755 --- a/dev/builddep-veloxbe-inc.sh +++ b/dev/builddep-veloxbe-inc.sh @@ -139,9 +139,11 @@ else export VCPKG_TRIPLET="x64-linux-avx" fi + export VCPKG_MANIFEST_DIR="$GLUTEN_DIR/dev/vcpkg" export VCPKG_TRIPLET_INSTALL_DIR="$GLUTEN_DIR/dev/vcpkg/vcpkg_installed/${VCPKG_TRIPLET}" export CMAKE_TOOLCHAIN_FILE="$GLUTEN_DIR/dev/vcpkg/toolchain.cmake" - export PKG_CONFIG_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH:-}" + unset PKG_CONFIG_PATH + export PKG_CONFIG_LIBDIR="${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig" export GLUTEN_VCPKG_ENABLED="${VCPKG_ROOT}" fi diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index f36be37377d..0e7b3a89bb2 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -45,6 +45,7 @@ VELOX_BRANCH="" VELOX_HOME="$GLUTEN_DIR/ep/build-velox/build/velox_ep" VELOX_PARAMETER="" BUILD_ARROW=ON +BUILD_ARROW_EXPLICIT=OFF SPARK_VERSION=ALL # set default number of threads as cpu cores minus 2 @@ -141,6 +142,7 @@ do ;; --build_arrow=*) BUILD_ARROW="${arg#*=}" + BUILD_ARROW_EXPLICIT=ON shift # Remove argument name from processing ;; --num_threads=*) @@ -158,6 +160,18 @@ do esac done +function vcpkg_is_active { + [ "$ENABLE_VCPKG" = "ON" ] || [ -n "${GLUTEN_VCPKG_ENABLED:-}" ] +} + +if vcpkg_is_active; then + if [ "$BUILD_ARROW_EXPLICIT" = "ON" ] && [ "$BUILD_ARROW" = "ON" ]; then + echo "ERROR: --build_arrow=ON is deprecated with --enable_vcpkg=ON; Arrow is managed by Gluten vcpkg." >&2 + exit 1 + fi + BUILD_ARROW=OFF +fi + if [[ "$(uname)" == "Darwin" ]]; then export INSTALL_PREFIX=${INSTALL_PREFIX:-${VELOX_HOME}/deps-install} if [[ "$INSTALL_PREFIX" == "/usr/local" || "$INSTALL_PREFIX" == /usr/local/* ]]; then @@ -223,6 +237,10 @@ concat_velox_param export VELOX_HOME function build_arrow { + if vcpkg_is_active; then + echo "ERROR: build_arrow is deprecated with --enable_vcpkg=ON; Arrow is managed by Gluten vcpkg." >&2 + return 1 + fi local GLUTEN_BUILD_TYPE="$BUILD_TYPE" if [ ! -d "$VELOX_HOME" ]; then get_velox @@ -273,8 +291,10 @@ function build_gluten_cpp { ) if [ -n "${INSTALL_PREFIX:-}" ]; then - GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX") GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX") + if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ]; then + GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX") + fi fi if [ $OS == 'Darwin' ]; then if [[ -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then diff --git a/dev/docker/Dockerfile.centos7-gcc13-static-build b/dev/docker/Dockerfile.centos7-gcc13-static-build index c9e6ede10e6..e351922af6d 100644 --- a/dev/docker/Dockerfile.centos7-gcc13-static-build +++ b/dev/docker/Dockerfile.centos7-gcc13-static-build @@ -43,10 +43,10 @@ RUN set -ex; \ yum remove gcc -y && yum clean all; \ yes | cp -rf /usr/share/aclocal/* /usr/local/share/aclocal/; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ cd /opt/gluten; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ dnf clean all; \ rm -rf /opt/gluten; \ diff --git a/dev/docker/Dockerfile.centos7-static-build b/dev/docker/Dockerfile.centos7-static-build index 638f144e0b5..fa3730035ca 100644 --- a/dev/docker/Dockerfile.centos7-static-build +++ b/dev/docker/Dockerfile.centos7-static-build @@ -39,12 +39,12 @@ RUN set -ex; \ echo "check_certificate = off" >> ~/.wgetrc; \ cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ cd /opt/gluten; \ source /opt/rh/devtoolset-11/enable; \ source /opt/rh/rh-git227/enable; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ dnf clean all; \ rm -rf /opt/gluten; \ diff --git a/dev/docker/Dockerfile.centos8-gcc13-static-build b/dev/docker/Dockerfile.centos8-gcc13-static-build index 37e609f64a1..9ee5f874587 100644 --- a/dev/docker/Dockerfile.centos8-gcc13-static-build +++ b/dev/docker/Dockerfile.centos8-gcc13-static-build @@ -37,14 +37,14 @@ RUN set -ex; \ dnf clean all; \ git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ if [ "$(uname -m)" = "aarch64" ]; then \ export CPU_TARGET="aarch64"; \ export VCPKG_FORCE_SYSTEM_BINARIES=1; \ fi; \ cd /opt/gluten; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ rm -rf /opt/gluten; \ rm -rf /root/.cache/ccache; diff --git a/dev/docker/Dockerfile.centos8-static-build b/dev/docker/Dockerfile.centos8-static-build index d6e2d5ec1c8..2f20ea64130 100644 --- a/dev/docker/Dockerfile.centos8-static-build +++ b/dev/docker/Dockerfile.centos8-static-build @@ -37,7 +37,7 @@ RUN set -ex; \ git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \ cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ if [ "$(uname -m)" = "aarch64" ]; then \ export CPU_TARGET="aarch64"; \ export VCPKG_FORCE_SYSTEM_BINARIES=1; \ @@ -45,8 +45,8 @@ RUN set -ex; \ source /opt/rh/gcc-toolset-11/enable; \ cd /opt/gluten; \ export SSL_VERIFY=false; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ rm -rf /opt/gluten; \ rm -rf /root/.cache/ccache; diff --git a/dev/docker/Dockerfile.centos9-static-build b/dev/docker/Dockerfile.centos9-static-build index c83c99a465a..5dbd4319111 100644 --- a/dev/docker/Dockerfile.centos9-static-build +++ b/dev/docker/Dockerfile.centos9-static-build @@ -33,15 +33,15 @@ RUN set -ex; \ git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \ cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \ mkdir -p ${VCPKG_PATH}; \ - echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ + echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \ if [ "$(uname -m)" = "aarch64" ]; then \ export CPU_TARGET="aarch64"; \ export VCPKG_FORCE_SYSTEM_BINARIES=1; \ fi; \ source /opt/rh/gcc-toolset-12/enable; \ cd /opt/gluten; \ - bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ - --enable_hdfs=ON --enable_abfs=ON build_arrow; \ + bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \ + --enable_hdfs=ON --enable_abfs=ON'; \ ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ rm -rf /opt/gluten; \ rm -rf /root/.cache/ccache; diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh index a86a953e2b5..a1b749800d9 100755 --- a/dev/vcpkg/env.sh +++ b/dev/vcpkg/env.sh @@ -29,6 +29,9 @@ export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg" export VCPKG_TRIPLET=$([ "${CPU_TARGET:-}" = "aarch64" ] && echo "arm64-linux-neon" || echo "x64-linux-avx") export VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET} +unset PKG_CONFIG_PATH +export PKG_CONFIG_LIBDIR=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig + ${SCRIPT_ROOT}/init.sh "$@" if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then @@ -45,9 +48,7 @@ if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then export VCPKG_ROOT=${VCPKG_ROOT} export VCPKG_MANIFEST_DIR=${SCRIPT_ROOT} export VCPKG_TRIPLET=${VCPKG_TRIPLET} - export CMAKE_TOOLCHAIN_FILE=${SCRIPT_ROOT}/toolchain.cmake - export PKG_CONFIG_PATH=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig:${PKG_CONFIG_PATH:-} export PATH="${EXPORT_TOOLS_PATH}:$PATH" export GLUTEN_VCPKG_ENABLED=${VCPKG_ROOT} diff --git a/dev/vcpkg/ports/arrow/android.patch b/dev/vcpkg/ports/arrow/android.patch new file mode 100644 index 00000000000..4eb412d1a52 --- /dev/null +++ b/dev/vcpkg/ports/arrow/android.patch @@ -0,0 +1,27 @@ +diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt +index 6dc8358..2b91efa 100644 +--- a/cpp/src/arrow/CMakeLists.txt ++++ b/cpp/src/arrow/CMakeLists.txt +@@ -166,7 +166,7 @@ if(WIN32) + list(APPEND ARROW_SYSTEM_LINK_LIBS "ws2_32") + endif() + +-if(NOT WIN32 AND NOT APPLE) ++if(NOT WIN32 AND NOT APPLE AND NOT ANDROID) + # Pass -lrt on Linux only + list(APPEND ARROW_SYSTEM_LINK_LIBS rt) + endif() +diff --git a/cpp/src/arrow/vendored/musl/strptime.c b/cpp/src/arrow/vendored/musl/strptime.c +index 41912fd..0ea36e9 100644 +--- a/cpp/src/arrow/vendored/musl/strptime.c ++++ b/cpp/src/arrow/vendored/musl/strptime.c +@@ -18,7 +18,9 @@ + #undef HAVE_LANGINFO + + #ifndef _WIN32 ++# if !(defined(__ANDROID__) && __ANDROID_API__ < 26) + #define HAVE_LANGINFO 1 ++# endif + #endif + + #ifdef HAVE_LANGINFO diff --git a/dev/vcpkg/ports/arrow/arrow-testing-boost.patch b/dev/vcpkg/ports/arrow/arrow-testing-boost.patch new file mode 100644 index 00000000000..a3ef9267565 --- /dev/null +++ b/dev/vcpkg/ports/arrow/arrow-testing-boost.patch @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This patch replaces Boost::process (header-only, no CMake target) with +# Boost::filesystem and Boost::system (the actual libraries it would have linked) +# Based on https://github.com/apache/arrow/pull/45424 + +diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +index db151b4..1da99f1 100644 +--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -1266,13 +1266,19 @@ endif() + # - Gandiva has a compile-time (header-only) dependency on Boost, not runtime. + # - Tests need Boost at runtime. + # - S3FS and Flight benchmarks need Boost at runtime. ++# - arrow_testing uses boost::filesystem. So arrow_testing requires ++# Boost library. (boost::filesystem isn't header-only.) But if we ++# use arrow_testing as a static library without ++# using arrow::util::Process, we don't need boost::filesystem. + if(ARROW_BUILD_INTEGRATION + OR ARROW_BUILD_TESTS + OR (ARROW_FLIGHT AND (ARROW_TESTING OR ARROW_BUILD_BENCHMARKS)) +- OR (ARROW_S3 AND ARROW_BUILD_BENCHMARKS)) ++ OR (ARROW_S3 AND ARROW_BUILD_BENCHMARKS) ++ OR (ARROW_TESTING AND ARROW_BUILD_SHARED)) + set(ARROW_USE_BOOST TRUE) + set(ARROW_BOOST_REQUIRE_LIBRARY TRUE) + elseif(ARROW_GANDIVA ++ OR ARROW_TESTING + OR ARROW_WITH_THRIFT + OR (NOT ARROW_USE_NATIVE_INT128)) + set(ARROW_USE_BOOST TRUE) +diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt +index c911f0f..84673d4 100644 +--- a/cpp/src/arrow/CMakeLists.txt ++++ b/cpp/src/arrow/CMakeLists.txt +@@ -645,11 +645,11 @@ else() + endif() + + set(ARROW_TESTING_SHARED_LINK_LIBS arrow_shared ${ARROW_GTEST_GTEST}) +-set(ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::flatbuffers RapidJSON Boost::process) ++set(ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::flatbuffers RapidJSON Boost::filesystem Boost::system) + set(ARROW_TESTING_STATIC_LINK_LIBS + arrow::flatbuffers + RapidJSON +- Boost::process ++ Boost::filesystem Boost::system + arrow_static + ${ARROW_GTEST_GTEST}) + set(ARROW_TESTING_SHARED_INSTALL_INTERFACE_LIBS Arrow::arrow_shared) diff --git a/dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch b/dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch new file mode 100644 index 00000000000..c87382699cb --- /dev/null +++ b/dev/vcpkg/ports/arrow/arrow-testing-static-boost.patch @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -1267,15 +1267,12 @@ endif() + # - Gandiva has a compile-time (header-only) dependency on Boost, not runtime. + # - Tests need Boost at runtime. + # - S3FS and Flight benchmarks need Boost at runtime. +-# - arrow_testing uses boost::filesystem. So arrow_testing requires +-# Boost library. (boost::filesystem isn't header-only.) But if we +-# use arrow_testing as a static library without +-# using arrow::util::Process, we don't need boost::filesystem. ++# - arrow_testing directly links Boost::filesystem and Boost::system. + if(ARROW_BUILD_INTEGRATION + OR ARROW_BUILD_TESTS + OR (ARROW_FLIGHT AND (ARROW_TESTING OR ARROW_BUILD_BENCHMARKS)) + OR (ARROW_S3 AND ARROW_BUILD_BENCHMARKS) +- OR (ARROW_TESTING AND ARROW_BUILD_SHARED)) ++ OR ARROW_TESTING) + set(ARROW_USE_BOOST TRUE) + set(ARROW_BOOST_REQUIRE_LIBRARY TRUE) + elseif(ARROW_GANDIVA diff --git a/dev/vcpkg/ports/arrow/msvc-static-name.patch b/dev/vcpkg/ports/arrow/msvc-static-name.patch new file mode 100644 index 00000000000..61d99ebac53 --- /dev/null +++ b/dev/vcpkg/ports/arrow/msvc-static-name.patch @@ -0,0 +1,13 @@ +diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake +index 391c43e0a..50f6d3d3c 100644 +--- a/cpp/cmake_modules/BuildUtils.cmake ++++ b/cpp/cmake_modules/BuildUtils.cmake +@@ -427,7 +427,7 @@ function(ADD_ARROW_LIB LIB_NAME) + target_include_directories(${LIB_NAME}_static PRIVATE ${ARG_PRIVATE_INCLUDES}) + endif() + +- if(MSVC_TOOLCHAIN) ++ if(MSVC_TOOLCHAIN AND 0) + set(LIB_NAME_STATIC ${LIB_NAME}_static) + else() + set(LIB_NAME_STATIC ${LIB_NAME}) diff --git a/dev/vcpkg/ports/arrow/portfile.cmake b/dev/vcpkg/ports/arrow/portfile.cmake new file mode 100644 index 00000000000..dfa795c81c4 --- /dev/null +++ b/dev/vcpkg/ports/arrow/portfile.cmake @@ -0,0 +1,152 @@ +vcpkg_download_distfile( + ARCHIVE_PATH + URLS "https://archive.apache.org/dist/arrow/arrow-${VERSION}/apache-arrow-${VERSION}.tar.gz" + FILENAME apache-arrow-${VERSION}.tar.gz + SHA512 4df30ab5561da695eaa864422626b9898555d86ca56835c3b8a8ca93a1dbaf081582bb36e2440d1daf7e1dd48c76941f1152a4f25ce0dbcc1c2abe244a00c05e +) +vcpkg_extract_source_archive( + SOURCE_PATH + ARCHIVE ${ARCHIVE_PATH} + PATCHES + android.patch + msvc-static-name.patch + utf8proc.patch + thrift.patch + arrow-testing-boost.patch + arrow-testing-static-boost.patch +) + +vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + acero ARROW_ACERO + compute ARROW_COMPUTE + csv ARROW_CSV + cuda ARROW_CUDA + dataset ARROW_DATASET + filesystem ARROW_FILESYSTEM + flight ARROW_FLIGHT + flightsql ARROW_FLIGHT_SQL + gcs ARROW_GCS + jemalloc ARROW_JEMALLOC + json ARROW_JSON + mimalloc ARROW_MIMALLOC + orc ARROW_ORC + parquet ARROW_PARQUET + parquet PARQUET_REQUIRE_ENCRYPTION + s3 ARROW_S3 + testing ARROW_TESTING +) + +if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) + list(APPEND FEATURE_OPTIONS "-DARROW_USE_NATIVE_INT128=OFF") +endif() + +string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "dynamic" ARROW_BUILD_SHARED) +string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "static" ARROW_BUILD_STATIC) +string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "dynamic" ARROW_DEPENDENCY_USE_SHARED) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}/cpp" + OPTIONS + ${FEATURE_OPTIONS} + -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED} + -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC} + -DARROW_BUILD_TESTS=OFF + -DARROW_DEPENDENCY_SOURCE=SYSTEM + -DARROW_DEPENDENCY_USE_SHARED=${ARROW_DEPENDENCY_USE_SHARED} + -DARROW_PACKAGE_KIND=vcpkg + -DARROW_WITH_BROTLI=OFF + -DARROW_WITH_BZ2=OFF + -DARROW_WITH_LZ4=ON + -DARROW_WITH_SNAPPY=ON + -DARROW_WITH_ZLIB=ON + -DARROW_WITH_ZSTD=ON + -DBUILD_WARNING_LEVEL=PRODUCTION + -DZSTD_MSVC_LIB_PREFIX= + MAYBE_UNUSED_VARIABLES + ZSTD_MSVC_LIB_PREFIX +) + +vcpkg_cmake_install() +vcpkg_copy_pdbs() + +vcpkg_fixup_pkgconfig() + +if(EXISTS "${CURRENT_PACKAGES_DIR}/lib/arrow_static.lib") + message(FATAL_ERROR "Installed lib file should be named 'arrow.lib' via patching the upstream build.") +endif() + +if("dataset" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME arrowdataset + CONFIG_PATH lib/cmake/ArrowDataset + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() + +if("acero" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME arrowacero + CONFIG_PATH lib/cmake/ArrowAcero + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() + +if("flight" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME ArrowFlight + CONFIG_PATH lib/cmake/ArrowFlight + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() + +if("flightsql" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME ArrowFlightSql + CONFIG_PATH lib/cmake/ArrowFlightSql + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() + +if("parquet" IN_LIST FEATURES) + vcpkg_cmake_config_fixup( + PACKAGE_NAME parquet + CONFIG_PATH lib/cmake/Parquet + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) +endif() +vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/Arrow) + +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +if("parquet" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-parquet" usage-parquet) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-parquet}") +endif() +if("dataset" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-dataset" usage-dataset) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-dataset}") +endif() +if("acero" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-acero" usage-acero) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-acero}") +endif() + +if("flight" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-flight" usage-flight) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-flight}") +endif() + +if("flightsql" IN_LIST FEATURES) + file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-flightsql" usage-flightsql) + file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${usage-flightsql}") +endif() + +if("example" IN_LIST FEATURES) + file(INSTALL "${SOURCE_PATH}/cpp/examples/minimal_build/" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}/example") +endif() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share/doc") + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE.txt") diff --git a/dev/vcpkg/ports/arrow/thrift.patch b/dev/vcpkg/ports/arrow/thrift.patch new file mode 100644 index 00000000000..ae284d65b85 --- /dev/null +++ b/dev/vcpkg/ports/arrow/thrift.patch @@ -0,0 +1,22 @@ +diff --git a/cpp/cmake_modules/FindThriftAlt.cmake b/cpp/cmake_modules/FindThriftAlt.cmake +index f3e4902..65ceac8 100644 +--- a/cpp/cmake_modules/FindThriftAlt.cmake ++++ b/cpp/cmake_modules/FindThriftAlt.cmake +@@ -45,7 +45,7 @@ endif() + # * https://github.com/apache/thrift/pull/2725 + # * https://github.com/apache/thrift/pull/2726 + # * https://github.com/conda-forge/thrift-cpp-feedstock/issues/68 +-if(NOT WIN32) ++ + set(find_package_args "") + if(ThriftAlt_FIND_VERSION) + list(APPEND find_package_args ${ThriftAlt_FIND_VERSION}) +@@ -61,7 +61,7 @@ if(NOT WIN32) + "${THRIFT_COMPILER}") + return() + endif() +-endif() ++ + + function(extract_thrift_version) + if(ThriftAlt_INCLUDE_DIR) diff --git a/dev/vcpkg/ports/arrow/usage b/dev/vcpkg/ports/arrow/usage new file mode 100644 index 00000000000..b07f1c9c32b --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage @@ -0,0 +1,4 @@ +The package arrow provides CMake targets: + + find_package(Arrow CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,Arrow::arrow_static,Arrow::arrow_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-acero b/dev/vcpkg/ports/arrow/usage-acero new file mode 100644 index 00000000000..05c4776b797 --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-acero @@ -0,0 +1,3 @@ + + find_package(ArrowAcero CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,ArrowAcero::arrow_acero_static,ArrowAcero::arrow_acero_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-dataset b/dev/vcpkg/ports/arrow/usage-dataset new file mode 100644 index 00000000000..8bedfb83246 --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-dataset @@ -0,0 +1,3 @@ + + find_package(ArrowDataset CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,ArrowDataset::arrow_dataset_static,ArrowDataset::arrow_dataset_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-flight b/dev/vcpkg/ports/arrow/usage-flight new file mode 100644 index 00000000000..c9656253f9e --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-flight @@ -0,0 +1,3 @@ + + find_package(ArrowFlight CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,ArrowFlight::arrow_flight_static,ArrowFlight::arrow_flight_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-flightsql b/dev/vcpkg/ports/arrow/usage-flightsql new file mode 100644 index 00000000000..3159c059036 --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-flightsql @@ -0,0 +1,3 @@ + + find_package(ArrowFlightSql CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,ArrowFlightSql::arrow_flight_sql_static,ArrowFlightSql::arrow_flight_sql_shared>") diff --git a/dev/vcpkg/ports/arrow/usage-parquet b/dev/vcpkg/ports/arrow/usage-parquet new file mode 100644 index 00000000000..ce04039a64e --- /dev/null +++ b/dev/vcpkg/ports/arrow/usage-parquet @@ -0,0 +1,3 @@ + + find_package(Parquet CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,Parquet::parquet_static,Parquet::parquet_shared>") diff --git a/dev/vcpkg/ports/arrow/utf8proc.patch b/dev/vcpkg/ports/arrow/utf8proc.patch new file mode 100644 index 00000000000..eea9ca59e2e --- /dev/null +++ b/dev/vcpkg/ports/arrow/utf8proc.patch @@ -0,0 +1,13 @@ +diff --git a/cpp/cmake_modules/Findutf8proc.cmake b/cpp/cmake_modules/Findutf8proc.cmake +index e347414..83f2aa1 100644 +--- a/cpp/cmake_modules/Findutf8proc.cmake ++++ b/cpp/cmake_modules/Findutf8proc.cmake +@@ -30,7 +30,7 @@ if(ARROW_PACKAGE_KIND STREQUAL "vcpkg") + if(utf8proc_FIND_REQUIRED) + list(APPEND find_package_args REQUIRED) + endif() +- find_package(utf8proc NAMES unofficial-utf8proc ${find_package_args}) ++ find_package(utf8proc NAMES unofficial-utf8proc) + if(utf8proc_FOUND) + add_library(utf8proc::utf8proc ALIAS utf8proc) + return() diff --git a/dev/vcpkg/ports/arrow/vcpkg.json b/dev/vcpkg/ports/arrow/vcpkg.json new file mode 100644 index 00000000000..f84b79ce3fd --- /dev/null +++ b/dev/vcpkg/ports/arrow/vcpkg.json @@ -0,0 +1,149 @@ +{ + "name": "arrow", + "version": "18.0.0", + "description": "Cross-language development platform for in-memory analytics", + "homepage": "https://arrow.apache.org", + "license": "Apache-2.0", + "supports": "x64 | (arm64 & !windows)", + "dependencies": [ + "boost-filesystem", + "boost-multiprecision", + "boost-system", + "brotli", + "bzip2", + "gflags", + "lz4", + "openssl", + "re2", + "snappy", + "thrift", + "utf8proc", + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + }, + "xsimd", + "zlib", + "zstd" + ], + "default-features": [ + "csv", + "filesystem", + "json", + "parquet" + ], + "features": { + "acero": { + "description": "Acero support" + }, + "compute": { + "description": "Build all computational kernel functions" + }, + "csv": { + "description": "CSV support" + }, + "cuda": { + "description": "cuda support", + "dependencies": [ + "cuda" + ] + }, + "dataset": { + "description": "Dataset support" + }, + "example": { + "description": "Install the minimal example (source code)" + }, + "filesystem": { + "description": "Filesystem support" + }, + "flight": { + "description": "Arrow Flight RPC support", + "dependencies": [ + "abseil", + "c-ares", + "grpc", + "protobuf" + ] + }, + "flightsql": { + "description": "FlightSQL support", + "dependencies": [ + { + "name": "arrow", + "default-features": false, + "features": [ + "flight" + ] + } + ] + }, + "gcs": { + "description": "GCS support", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "storage" + ] + } + ] + }, + "jemalloc": { + "description": "jemalloc allocator", + "supports": "!windows" + }, + "json": { + "description": "JSON support", + "dependencies": [ + "rapidjson" + ] + }, + "mimalloc": { + "description": "mimalloc allocator", + "supports": "windows" + }, + "orc": { + "description": "ORC support", + "dependencies": [ + "orc" + ] + }, + "parquet": { + "description": "Parquet support", + "dependencies": [ + "rapidjson" + ] + }, + "s3": { + "description": "S3 support", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "cognito-identity", + "config", + "identity-management", + "s3", + "sts", + "transfer" + ] + } + ] + }, + "testing": { + "description": "Build the Arrow testing support library", + "dependencies": [ + "boost-process", + "gtest", + "rapidjson" + ] + } + } +} diff --git a/dev/vcpkg/toolchain.cmake b/dev/vcpkg/toolchain.cmake index f4db8e7af53..fda971feb49 100644 --- a/dev/vcpkg/toolchain.cmake +++ b/dev/vcpkg/toolchain.cmake @@ -33,11 +33,36 @@ set(VCPKG_HOST_TRIPLET $ENV{VCPKG_TRIPLET}) set(VCPKG_INSTALLED_DIR $ENV{VCPKG_MANIFEST_DIR}/vcpkg_installed) set(VCPKG_INSTALL_OPTIONS --no-print-usage) -# Force read CMAKE_PREFIX_PATH from env -set(CMAKE_PREFIX_PATH $ENV{CMAKE_PREFIX_PATH}) - include($ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake) +if(DEFINED ENV{GLUTEN_VCPKG_PREFER_CONFIG} + AND NOT "$ENV{GLUTEN_VCPKG_PREFER_CONFIG}" STREQUAL "") + set(_GLUTEN_VCPKG_PREFER_CONFIG "$ENV{GLUTEN_VCPKG_PREFER_CONFIG}") +elseif(DEFINED CMAKE_FIND_PACKAGE_PREFER_CONFIG) + set(_GLUTEN_VCPKG_PREFER_CONFIG "${CMAKE_FIND_PACKAGE_PREFER_CONFIG}") +else() + set(_GLUTEN_VCPKG_PREFER_CONFIG ON) +endif() +set(CMAKE_FIND_PACKAGE_PREFER_CONFIG "${_GLUTEN_VCPKG_PREFER_CONFIG}" CACHE BOOL + "Prefer package configuration files." FORCE) +unset(_GLUTEN_VCPKG_PREFER_CONFIG) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY CACHE STRING "Search for packages only in root paths." FORCE) +set(CMAKE_FIND_USE_PACKAGE_REGISTRY OFF CACHE BOOL "Disable the user package registry." FORCE) +set(CMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY OFF CACHE BOOL "Disable the system package registry." FORCE) + +set(_GLUTEN_VCPKG_IGNORED_PREFIXES /usr/local) +if(APPLE) + list(APPEND _GLUTEN_VCPKG_IGNORED_PREFIXES /opt/homebrew /opt/local) +endif() +if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "") + list(APPEND _GLUTEN_VCPKG_IGNORED_PREFIXES "$ENV{CONDA_PREFIX}") +endif() +list(APPEND CMAKE_IGNORE_PREFIX_PATH ${_GLUTEN_VCPKG_IGNORED_PREFIXES}) +list(REMOVE_DUPLICATES CMAKE_IGNORE_PREFIX_PATH) +set(CMAKE_IGNORE_PREFIX_PATH "${CMAKE_IGNORE_PREFIX_PATH}" CACHE STRING + "Prefixes ignored by Gluten's vcpkg toolchain." FORCE) +unset(_GLUTEN_VCPKG_IGNORED_PREFIXES) + set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -static-libgcc") set(CMAKE_SHARED_LINKER_FLAGS "-static-libstdc++ -static-libgcc") diff --git a/dev/vcpkg/vcpkg-configuration.json b/dev/vcpkg/vcpkg-configuration.json index ba7c7f58b32..a7aa80d84ec 100644 --- a/dev/vcpkg/vcpkg-configuration.json +++ b/dev/vcpkg/vcpkg-configuration.json @@ -3,6 +3,12 @@ "overlay-ports": [ "./ports" ], "overlay-triplets": [ "./triplets" ], "registries": [ + { + "kind": "git", + "repository": "https://github.com/Microsoft/vcpkg", + "baseline": "bcf3d00d2116056fda0ce47615f6074ffecb7524", + "packages": [ "boost", "boost-*"] + }, { "kind": "git", "repository": "https://github.com/Microsoft/vcpkg", diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json index 55cf7573644..8731204d193 100644 --- a/dev/vcpkg/vcpkg.json +++ b/dev/vcpkg/vcpkg.json @@ -12,6 +12,11 @@ "lz4", "double-conversion", "bzip2", + { + "name": "arrow", + "default-features": false, + "features": ["testing"] + }, "curl", "lzo", "re2", diff --git a/ep/build-velox/src/build-velox.sh b/ep/build-velox/src/build-velox.sh index 3e279bf6808..a8f4d806dad 100755 --- a/ep/build-velox/src/build-velox.sh +++ b/ep/build-velox/src/build-velox.sh @@ -114,7 +114,7 @@ function compile { -Wno-error=uninitialized -Wno-unknown-warning-option -Wno-deprecated-declarations' if [[ "$(uname)" == "Darwin" ]]; then CXX_FLAGS="$CXX_FLAGS -Wno-inconsistent-missing-override -Wno-macro-redefined" - if [[ -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then + if [[ -z "${GLUTEN_VCPKG_ENABLED:-}" && -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then # Add the dependency prefix as a system include: this finds deps that only # publish loose headers (e.g. xsimd) and demotes warnings in vendored # dependency headers (abseil's __is_trivially_relocatable, arrow's vendored @@ -128,7 +128,10 @@ function compile { -DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_RUNNER=OFF -DVELOX_SIMDJSON_SKIPUTF8VALIDATION=ON \ -DVELOX_ENABLE_GEO=OFF" if [ -n "${INSTALL_PREFIX:-}" ]; then - COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" + COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" + if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ]; then + COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX}" + fi fi if [[ "$(uname)" == "Darwin" && -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PREFIX_PATH=/usr/local" @@ -166,6 +169,14 @@ function compile { -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc" fi if [ -n "${GLUTEN_VCPKG_ENABLED:-}" ]; then + # Flex is a host build tool, so admit only its matching executable and headers. + FLEX_EXECUTABLE="$(realpath "$(command -v flex)")" + FLEX_INCLUDE_DIR="$(dirname "$(dirname "${FLEX_EXECUTABLE}")")/include" + if [ ! -f "${FLEX_INCLUDE_DIR}/FlexLexer.h" ]; then + echo "FlexLexer.h not found for ${FLEX_EXECUTABLE}: ${FLEX_INCLUDE_DIR}" >&2 + return 1 + fi + COMPILE_OPTION="$COMPILE_OPTION -DFLEX_EXECUTABLE=${FLEX_EXECUTABLE} -DFLEX_INCLUDE_DIR=${FLEX_INCLUDE_DIR}" COMPILE_OPTION="$COMPILE_OPTION -DVELOX_GFLAGS_TYPE=static" fi @@ -179,12 +190,20 @@ function compile { fi echo "NUM_THREADS_OPTS: $NUM_THREADS_OPTS" - export simdjson_SOURCE=AUTO - export Arrow_SOURCE=AUTO + local -a VELOX_MAKE_COMMAND=(make) + if [ -n "${GLUTEN_VCPKG_ENABLED:-}" ]; then + export VELOX_DEPENDENCY_SOURCE=SYSTEM + export simdjson_SOURCE=SYSTEM + export Arrow_SOURCE=SYSTEM + VELOX_MAKE_COMMAND=(env GLUTEN_VCPKG_PREFER_CONFIG=OFF make) + else + export simdjson_SOURCE=AUTO + export Arrow_SOURCE=AUTO + fi if [ $ARCH == 'x86_64' ]; then - make $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" + "${VELOX_MAKE_COMMAND[@]}" $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" elif [[ "$ARCH" == 'arm64' || "$ARCH" == 'aarch64' || "$ARCH" == "ppc64le" ]]; then - CPU_TARGET=$ARCH make $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" + CPU_TARGET=$ARCH "${VELOX_MAKE_COMMAND[@]}" $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" else echo "Unsupported arch: $ARCH" exit 1 diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 72865dc94a6..7f27e708d5a 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -153,11 +153,7 @@ function apply_provided_velox_patch { } function apply_compilation_fixes { - local SUDO_CMD="" - if [ "$OS" == "Linux" ]; then - SUDO_CMD="sudo" - fi - $SUDO_CMD cp ${CURRENT_DIR}/modify_arrow.patch ${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/ + cp ${CURRENT_DIR}/modify_arrow.patch ${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/ git add ${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/modify_arrow.patch # to avoid the file from being deleted by git clean -dffx :/ } diff --git a/tools/workload/benchmark_velox/initialize.ipynb b/tools/workload/benchmark_velox/initialize.ipynb index 1d3e6397b6b..be106993aab 100644 --- a/tools/workload/benchmark_velox/initialize.ipynb +++ b/tools/workload/benchmark_velox/initialize.ipynb @@ -2232,18 +2232,6 @@ " !git clone https://github.com/apache/gluten.git gluten" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "hidden": true - }, - "outputs": [], - "source": [ - "# Build Arrow for the first time build.\n", - "!sed -i 's/--build_arrow=OFF/--build_arrow=ON/' ~/gluten/dev/package-vcpkg.sh" - ] - }, { "cell_type": "code", "execution_count": null,