diff --git a/cmake_modules/DefineOptions.cmake b/cmake_modules/DefineOptions.cmake index 0211f409..b759feca 100644 --- a/cmake_modules/DefineOptions.cmake +++ b/cmake_modules/DefineOptions.cmake @@ -127,6 +127,53 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(PAIMON_USE_TSAN "Enable Thread Sanitizer checks" OFF) define_option(PAIMON_USE_UBSAN "Enable Undefined Behaviour Sanitizer checks" OFF) + #---------------------------------------------------------------------- + set_option_category("Thirdparty dependencies") + + if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "") + set(PAIMON_DEPENDENCY_SOURCE_DEFAULT "CONDA") + else() + set(PAIMON_DEPENDENCY_SOURCE_DEFAULT "AUTO") + endif() + + define_option_string(PAIMON_DEPENDENCY_SOURCE + "Method to use for acquiring Paimon's build dependencies" + "${PAIMON_DEPENDENCY_SOURCE_DEFAULT}" + "AUTO" + "BUNDLED" + "SYSTEM" + "CONDA" + "VCPKG" + "BREW") + + define_option(PAIMON_VERBOSE_THIRDPARTY_BUILD + "Show output from ExternalProjects rather than just logging to files" + OFF) + + define_option(PAIMON_DEPENDENCY_USE_SHARED + "Link to shared libraries for dependencies where available" + ON) + + define_option(PAIMON_SNAPPY_USE_SHARED + "Rely on Snappy shared libraries where relevant" + ${PAIMON_DEPENDENCY_USE_SHARED}) + + define_option(PAIMON_ZSTD_USE_SHARED + "Rely on zstd shared libraries where relevant" + ${PAIMON_DEPENDENCY_USE_SHARED}) + + define_option(PAIMON_LZ4_USE_SHARED + "Rely on lz4 shared libraries where relevant" + ${PAIMON_DEPENDENCY_USE_SHARED}) + + define_option(PAIMON_ZLIB_USE_SHARED + "Rely on zlib shared libraries where relevant" + ${PAIMON_DEPENDENCY_USE_SHARED}) + + define_option(PAIMON_GLOG_USE_SHARED + "Rely on glog shared libraries where relevant" + ${PAIMON_DEPENDENCY_USE_SHARED}) + #---------------------------------------------------------------------- set_option_category("Advanced developer") diff --git a/cmake_modules/FindSnappyAlt.cmake b/cmake_modules/FindSnappyAlt.cmake new file mode 100644 index 00000000..45ff9cc6 --- /dev/null +++ b/cmake_modules/FindSnappyAlt.cmake @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(SnappyAlt_FOUND) + return() +endif() + +set(find_package_args) +if(SnappyAlt_FIND_VERSION) + list(APPEND find_package_args ${SnappyAlt_FIND_VERSION}) +endif() +if(SnappyAlt_FIND_QUIETLY) + list(APPEND find_package_args QUIET) +endif() + +# Try to find Snappy using standard find_package first +find_package(Snappy ${find_package_args}) +if(Snappy_FOUND) + if(PAIMON_SNAPPY_USE_SHARED) + set(Snappy_TARGET Snappy::snappy) + set(SnappyAlt_FOUND TRUE) + return() + else() + if(TARGET Snappy::snappy-static) + set(Snappy_TARGET Snappy::snappy-static) + set(SnappyAlt_FOUND TRUE) + return() + else() + get_target_property(Snappy_TYPE Snappy::snappy TYPE) + if(Snappy_TYPE STREQUAL "STATIC_LIBRARY" OR Snappy_TYPE STREQUAL "INTERFACE_LIBRARY") + set(Snappy_TARGET Snappy::snappy) + set(SnappyAlt_FOUND TRUE) + return() + endif() + endif() + endif() +endif() + +# Manual search if standard find_package didn't work +set(PAIMON_LIBRARY_PATH_SUFFIXES + lib + lib64 + lib/${CMAKE_LIBRARY_ARCHITECTURE}) +set(PAIMON_INCLUDE_PATH_SUFFIXES include) + +if(PAIMON_SNAPPY_USE_SHARED) + set(SNAPPY_LIB_NAMES) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list(APPEND SNAPPY_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}snappy${CMAKE_IMPORT_LIBRARY_SUFFIX}") + endif() + list(APPEND SNAPPY_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}snappy${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + set(SNAPPY_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}snappy${CMAKE_STATIC_LIBRARY_SUFFIX}") +endif() + +if(Snappy_ROOT) + find_library(Snappy_LIB + NAMES ${SNAPPY_LIB_NAMES} + PATHS ${Snappy_ROOT} + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES} + NO_DEFAULT_PATH) + find_path(Snappy_INCLUDE_DIR + NAMES snappy.h + PATHS ${Snappy_ROOT} + NO_DEFAULT_PATH + PATH_SUFFIXES ${PAIMON_INCLUDE_PATH_SUFFIXES}) +else() + find_library(Snappy_LIB NAMES ${SNAPPY_LIB_NAMES} + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES}) + find_path(Snappy_INCLUDE_DIR + NAMES snappy.h + PATH_SUFFIXES ${PAIMON_INCLUDE_PATH_SUFFIXES}) +endif() + +find_package_handle_standard_args(SnappyAlt REQUIRED_VARS Snappy_LIB Snappy_INCLUDE_DIR) + +if(SnappyAlt_FOUND) + if(PAIMON_SNAPPY_USE_SHARED) + set(Snappy_TARGET Snappy::snappy) + set(Snappy_TARGET_TYPE SHARED) + else() + set(Snappy_TARGET Snappy::snappy) + set(Snappy_TARGET_TYPE STATIC) + endif() + + if(NOT TARGET ${Snappy_TARGET}) + add_library(${Snappy_TARGET} ${Snappy_TARGET_TYPE} IMPORTED) + set_target_properties(${Snappy_TARGET} + PROPERTIES IMPORTED_LOCATION "${Snappy_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${Snappy_INCLUDE_DIR}") + endif() +endif() diff --git a/cmake_modules/FindglogAlt.cmake b/cmake_modules/FindglogAlt.cmake new file mode 100644 index 00000000..8ee28c51 --- /dev/null +++ b/cmake_modules/FindglogAlt.cmake @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(glogAlt_FOUND) + return() +endif() + +set(find_package_args) +if(glogAlt_FIND_VERSION) + list(APPEND find_package_args ${glogAlt_FIND_VERSION}) +endif() +if(glogAlt_FIND_QUIETLY) + list(APPEND find_package_args QUIET) +endif() + +# Try to find glog using standard find_package first +find_package(glog ${find_package_args}) +if(glog_FOUND) + set(glogAlt_FOUND TRUE) + if(TARGET glog::glog) + return() + endif() +endif() + +# Manual search +set(PAIMON_LIBRARY_PATH_SUFFIXES + lib + lib64 + lib/${CMAKE_LIBRARY_ARCHITECTURE}) +set(PAIMON_INCLUDE_PATH_SUFFIXES include) + +if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(GLOG_LIB_SUFFIX "d") +else() + set(GLOG_LIB_SUFFIX "") +endif() + +if(PAIMON_GLOG_USE_SHARED) + set(GLOG_LIB_NAMES) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list(APPEND GLOG_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}glog${GLOG_LIB_SUFFIX}${CMAKE_IMPORT_LIBRARY_SUFFIX}") + endif() + list(APPEND GLOG_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}glog${GLOG_LIB_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + set(GLOG_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}glog${GLOG_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") +endif() + +if(glog_ROOT) + find_library(glog_LIB + NAMES ${GLOG_LIB_NAMES} + PATHS ${glog_ROOT} + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES} + NO_DEFAULT_PATH) + find_path(glog_INCLUDE_DIR + NAMES glog/logging.h + PATHS ${glog_ROOT} + NO_DEFAULT_PATH + PATH_SUFFIXES ${PAIMON_INCLUDE_PATH_SUFFIXES}) +else() + find_library(glog_LIB NAMES ${GLOG_LIB_NAMES} + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES}) + find_path(glog_INCLUDE_DIR + NAMES glog/logging.h + PATH_SUFFIXES ${PAIMON_INCLUDE_PATH_SUFFIXES}) +endif() + +find_package_handle_standard_args(glogAlt REQUIRED_VARS glog_LIB glog_INCLUDE_DIR) + +if(glogAlt_FOUND) + if(NOT TARGET glog::glog) + if(PAIMON_GLOG_USE_SHARED) + set(glog_TARGET_TYPE SHARED) + else() + set(glog_TARGET_TYPE STATIC) + endif() + add_library(glog::glog ${glog_TARGET_TYPE} IMPORTED) + set_target_properties(glog::glog + PROPERTIES IMPORTED_LOCATION "${glog_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${glog_INCLUDE_DIR}" + INTERFACE_COMPILE_DEFINITIONS "GLOG_USE_GLOG_EXPORT") + endif() +endif() diff --git a/cmake_modules/Findlz4Alt.cmake b/cmake_modules/Findlz4Alt.cmake new file mode 100644 index 00000000..b640d62d --- /dev/null +++ b/cmake_modules/Findlz4Alt.cmake @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(lz4Alt_FOUND) + return() +endif() + +set(find_package_args) +if(lz4Alt_FIND_VERSION) + list(APPEND find_package_args ${lz4Alt_FIND_VERSION}) +endif() +if(lz4Alt_FIND_QUIETLY) + list(APPEND find_package_args QUIET) +endif() + +# Try to find lz4 using standard find_package first +find_package(lz4 ${find_package_args}) +if(lz4_FOUND) + set(lz4Alt_FOUND TRUE) + if(TARGET LZ4::lz4) + return() + elseif(TARGET lz4::lz4) + add_library(LZ4::lz4 ALIAS lz4::lz4) + return() + elseif(PAIMON_LZ4_USE_SHARED AND TARGET LZ4::lz4_shared) + add_library(LZ4::lz4 ALIAS LZ4::lz4_shared) + return() + elseif(TARGET LZ4::lz4_static) + add_library(LZ4::lz4 ALIAS LZ4::lz4_static) + return() + endif() +endif() + +# Manual search +set(PAIMON_LIBRARY_PATH_SUFFIXES + lib + lib64 + lib/${CMAKE_LIBRARY_ARCHITECTURE}) +set(PAIMON_INCLUDE_PATH_SUFFIXES include) + +if(MSVC_TOOLCHAIN AND NOT DEFINED LZ4_MSVC_LIB_PREFIX) + set(LZ4_MSVC_LIB_PREFIX "lib") +endif() +set(LZ4_LIB_NAME_BASE "${LZ4_MSVC_LIB_PREFIX}lz4") + +if(PAIMON_LZ4_USE_SHARED) + set(LZ4_LIB_NAMES) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list(APPEND LZ4_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}${LZ4_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}") + endif() + list(APPEND LZ4_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}${LZ4_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + if(MSVC AND NOT DEFINED LZ4_MSVC_STATIC_LIB_SUFFIX) + set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static") + endif() + set(LZ4_STATIC_LIB_SUFFIX "${LZ4_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(LZ4_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}${LZ4_LIB_NAME_BASE}${LZ4_STATIC_LIB_SUFFIX}") +endif() + +if(lz4_ROOT) + find_library(LZ4_LIB + NAMES ${LZ4_LIB_NAMES} + PATHS ${lz4_ROOT} + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES} + NO_DEFAULT_PATH) + find_path(LZ4_INCLUDE_DIR + NAMES lz4.h + PATHS ${lz4_ROOT} + NO_DEFAULT_PATH + PATH_SUFFIXES ${PAIMON_INCLUDE_PATH_SUFFIXES}) +else() + # Try pkg-config first + find_package(PkgConfig QUIET) + pkg_check_modules(LZ4_PC liblz4 QUIET) + if(LZ4_PC_FOUND) + set(LZ4_INCLUDE_DIR "${LZ4_PC_INCLUDEDIR}") + list(APPEND LZ4_PC_LIBRARY_DIRS "${LZ4_PC_LIBDIR}") + find_library(LZ4_LIB + NAMES ${LZ4_LIB_NAMES} + PATHS ${LZ4_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES}) + else() + find_library(LZ4_LIB NAMES ${LZ4_LIB_NAMES} + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES}) + find_path(LZ4_INCLUDE_DIR + NAMES lz4.h + PATH_SUFFIXES ${PAIMON_INCLUDE_PATH_SUFFIXES}) + endif() +endif() + +find_package_handle_standard_args(lz4Alt REQUIRED_VARS LZ4_LIB LZ4_INCLUDE_DIR) + +if(lz4Alt_FOUND) + if(NOT TARGET LZ4::lz4) + add_library(LZ4::lz4 UNKNOWN IMPORTED) + set_target_properties(LZ4::lz4 + PROPERTIES IMPORTED_LOCATION "${LZ4_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}") + endif() +endif() diff --git a/cmake_modules/FindzstdAlt.cmake b/cmake_modules/FindzstdAlt.cmake new file mode 100644 index 00000000..81f52478 --- /dev/null +++ b/cmake_modules/FindzstdAlt.cmake @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(zstdAlt_FOUND) + return() +endif() + +set(find_package_args) +if(zstdAlt_FIND_VERSION) + list(APPEND find_package_args ${zstdAlt_FIND_VERSION}) +endif() +if(zstdAlt_FIND_QUIETLY) + list(APPEND find_package_args QUIET) +endif() + +# Try to find zstd using standard find_package first +find_package(zstd ${find_package_args}) +if(zstd_FOUND) + set(zstdAlt_FOUND TRUE) + if(PAIMON_ZSTD_USE_SHARED) + if(TARGET zstd::libzstd_shared) + set(zstd_TARGET zstd::libzstd_shared) + elseif(TARGET zstd::libzstd) + set(zstd_TARGET zstd::libzstd) + endif() + else() + if(TARGET zstd::libzstd_static) + set(zstd_TARGET zstd::libzstd_static) + elseif(TARGET zstd::libzstd) + set(zstd_TARGET zstd::libzstd) + endif() + endif() + if(zstd_TARGET) + return() + endif() +endif() + +# Manual search +set(PAIMON_LIBRARY_PATH_SUFFIXES + lib + lib64 + lib/${CMAKE_LIBRARY_ARCHITECTURE}) +set(PAIMON_INCLUDE_PATH_SUFFIXES include) + +if(PAIMON_ZSTD_USE_SHARED) + set(ZSTD_LIB_NAMES) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list(APPEND ZSTD_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}zstd${CMAKE_IMPORT_LIBRARY_SUFFIX}") + endif() + list(APPEND ZSTD_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}zstd${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + set(ZSTD_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}zstd${CMAKE_STATIC_LIBRARY_SUFFIX}") +endif() + +if(zstd_ROOT) + find_library(zstd_LIB + NAMES ${ZSTD_LIB_NAMES} + PATHS ${zstd_ROOT} + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES} + NO_DEFAULT_PATH) + find_path(zstd_INCLUDE_DIR + NAMES zstd.h + PATHS ${zstd_ROOT} + NO_DEFAULT_PATH + PATH_SUFFIXES ${PAIMON_INCLUDE_PATH_SUFFIXES}) +else() + # Try pkg-config first + find_package(PkgConfig QUIET) + pkg_check_modules(zstd_PC libzstd QUIET) + if(zstd_PC_FOUND) + set(zstd_INCLUDE_DIR "${zstd_PC_INCLUDEDIR}") + list(APPEND zstd_PC_LIBRARY_DIRS "${zstd_PC_LIBDIR}") + find_library(zstd_LIB + NAMES ${ZSTD_LIB_NAMES} + PATHS ${zstd_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES}) + else() + find_library(zstd_LIB NAMES ${ZSTD_LIB_NAMES} + PATH_SUFFIXES ${PAIMON_LIBRARY_PATH_SUFFIXES}) + find_path(zstd_INCLUDE_DIR + NAMES zstd.h + PATH_SUFFIXES ${PAIMON_INCLUDE_PATH_SUFFIXES}) + endif() +endif() + +find_package_handle_standard_args(zstdAlt REQUIRED_VARS zstd_LIB zstd_INCLUDE_DIR) + +if(zstdAlt_FOUND) + if(PAIMON_ZSTD_USE_SHARED) + set(zstd_TARGET zstd::libzstd_shared) + set(zstd_TARGET_TYPE SHARED) + else() + set(zstd_TARGET zstd::libzstd_static) + set(zstd_TARGET_TYPE STATIC) + endif() + + if(NOT TARGET ${zstd_TARGET}) + add_library(${zstd_TARGET} ${zstd_TARGET_TYPE} IMPORTED) + set_target_properties(${zstd_TARGET} + PROPERTIES IMPORTED_LOCATION "${zstd_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${zstd_INCLUDE_DIR}") + endif() +endif() diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 511a6919..aa876dca 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -15,6 +15,56 @@ # specific language governing permissions and limitations # under the License. +# ---------------------------------------------------------------------- +# Dependency resolution control + +set(PAIMON_THIRDPARTY_DEPENDENCIES + Arrow + Snappy + zstd + lz4 + zlib + fmt + glog + RapidJSON + tbb + Protobuf + avro + orc) + +message(STATUS "Using ${PAIMON_DEPENDENCY_SOURCE} approach to find dependencies") + +if(PAIMON_DEPENDENCY_SOURCE STREQUAL "CONDA") + if(MSVC) + set(PAIMON_PACKAGE_PREFIX "$ENV{CONDA_PREFIX}/Library") + else() + set(PAIMON_PACKAGE_PREFIX $ENV{CONDA_PREFIX}) + endif() + set(PAIMON_ACTUAL_DEPENDENCY_SOURCE "SYSTEM") + message(STATUS "Using CONDA_PREFIX for PAIMON_PACKAGE_PREFIX: ${PAIMON_PACKAGE_PREFIX}") +else() + set(PAIMON_ACTUAL_DEPENDENCY_SOURCE "${PAIMON_DEPENDENCY_SOURCE}") +endif() + +if(PAIMON_PACKAGE_PREFIX) + message(STATUS "Setting (unset) dependency *_ROOT variables: ${PAIMON_PACKAGE_PREFIX}") + set(ENV{PKG_CONFIG_PATH} "${PAIMON_PACKAGE_PREFIX}/lib/pkgconfig/") +endif() + +# For each dependency, set dependency source to global default, if unset +foreach(DEPENDENCY ${PAIMON_THIRDPARTY_DEPENDENCIES}) + if("${${DEPENDENCY}_SOURCE}" STREQUAL "") + set(${DEPENDENCY}_SOURCE ${PAIMON_ACTUAL_DEPENDENCY_SOURCE}) + # If no ROOT was supplied and we have a global prefix, use it + if(NOT ${DEPENDENCY}_ROOT AND PAIMON_PACKAGE_PREFIX) + set(${DEPENDENCY}_ROOT ${PAIMON_PACKAGE_PREFIX}) + endif() + endif() +endforeach() + +# ---------------------------------------------------------------------- +# Thirdparty build configuration + set(THIRDPARTY_LOG_OPTIONS LOG_CONFIGURE 1 @@ -287,6 +337,92 @@ set(EP_COMMON_CMAKE_ARGS -DCMAKE_C_FLAGS=${EP_C_FLAGS} -DCMAKE_INSTALL_LIBDIR=lib) +# ---------------------------------------------------------------------- +# Dependency resolution macros + +macro(build_dependency DEPENDENCY_NAME) + if("${DEPENDENCY_NAME}" STREQUAL "Snappy") + build_snappy() + elseif("${DEPENDENCY_NAME}" STREQUAL "zstd") + build_zstd() + elseif("${DEPENDENCY_NAME}" STREQUAL "lz4") + build_lz4() + elseif("${DEPENDENCY_NAME}" STREQUAL "zlib") + build_zlib() + elseif("${DEPENDENCY_NAME}" STREQUAL "fmt") + build_fmt() + elseif("${DEPENDENCY_NAME}" STREQUAL "glog") + build_glog() + elseif("${DEPENDENCY_NAME}" STREQUAL "RapidJSON") + build_rapidjson() + elseif("${DEPENDENCY_NAME}" STREQUAL "Protobuf") + build_protobuf() + elseif("${DEPENDENCY_NAME}" STREQUAL "avro") + build_avro() + elseif("${DEPENDENCY_NAME}" STREQUAL "orc") + build_orc() + elseif("${DEPENDENCY_NAME}" STREQUAL "Arrow") + build_arrow() + elseif("${DEPENDENCY_NAME}" STREQUAL "tbb") + build_tbb() + elseif("${DEPENDENCY_NAME}" STREQUAL "Boost") + build_boost() + elseif("${DEPENDENCY_NAME}" STREQUAL "lucene") + build_lucene() + else() + message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") + endif() +endmacro() + +macro(resolve_dependency DEPENDENCY_NAME) + set(options) + set(one_value_args + HAVE_ALT + REQUIRED_VERSION + USE_CONFIG) + set(multi_value_args PC_PACKAGE_NAMES) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + if(ARG_HAVE_ALT) + set(PACKAGE_NAME "${DEPENDENCY_NAME}Alt") + else() + set(PACKAGE_NAME ${DEPENDENCY_NAME}) + endif() + + set(FIND_PACKAGE_ARGUMENTS ${PACKAGE_NAME}) + if(ARG_REQUIRED_VERSION) + list(APPEND FIND_PACKAGE_ARGUMENTS ${ARG_REQUIRED_VERSION}) + endif() + if(ARG_USE_CONFIG) + list(APPEND FIND_PACKAGE_ARGUMENTS CONFIG) + endif() + + if(${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") + find_package(${FIND_PACKAGE_ARGUMENTS}) + if(${PACKAGE_NAME}_FOUND) + message(STATUS "Found ${DEPENDENCY_NAME} (system): ${${PACKAGE_NAME}_VERSION}") + set(${DEPENDENCY_NAME}_SOURCE "SYSTEM") + else() + message(STATUS "Building ${DEPENDENCY_NAME} from source") + build_dependency(${DEPENDENCY_NAME}) + set(${DEPENDENCY_NAME}_SOURCE "BUNDLED") + endif() + elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") + message(STATUS "Building ${DEPENDENCY_NAME} from source (forced)") + build_dependency(${DEPENDENCY_NAME}) + elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") + find_package(${FIND_PACKAGE_ARGUMENTS} REQUIRED) + message(STATUS "Found ${DEPENDENCY_NAME} (system): ${${PACKAGE_NAME}_VERSION}") + endif() +endmacro() + macro(build_lucene) message(STATUS "Building lucene from source") set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install") @@ -1248,24 +1384,41 @@ macro(build_glog) endif() endmacro() -build_fmt() -build_rapidjson() -build_snappy() -build_zstd() -build_zlib() -build_lz4() -build_arrow() -build_tbb() -build_glog() +# ---------------------------------------------------------------------- +# Resolve all dependencies + +resolve_dependency(fmt) +resolve_dependency(RapidJSON) + +resolve_dependency(Snappy + HAVE_ALT TRUE) + +resolve_dependency(zstd + HAVE_ALT TRUE + REQUIRED_VERSION 1.4.0) + +resolve_dependency(lz4 + HAVE_ALT TRUE) + +resolve_dependency(zlib) + +resolve_dependency(glog + HAVE_ALT TRUE) + +resolve_dependency(tbb) +resolve_dependency(Arrow) if(PAIMON_ENABLE_AVRO) - build_avro() + resolve_dependency(avro) endif() + if(PAIMON_ENABLE_ORC) - build_protobuf() - build_orc() + resolve_dependency(Protobuf) + resolve_dependency(orc) endif() + if(PAIMON_ENABLE_JINDO) + # Jindo SDK still uses direct build as it's a precompiled package build_jindosdk_c() build_jindosdk_nextarch() endif() diff --git a/docs/DependencyManagement.md b/docs/DependencyManagement.md new file mode 100644 index 00000000..87e338f3 --- /dev/null +++ b/docs/DependencyManagement.md @@ -0,0 +1,248 @@ +# Dependency Management + +Paimon C++ provides flexible dependency management to support different build environments and workflows. + +## Quick Start + +### Use System Libraries (default: AUTO) + +By default, Paimon will try to find system libraries first, and fall back to building from source if not found: + +```bash +cmake -B build +``` + +### Force Building All Dependencies from Source + +```bash +cmake -B build -DPAIMON_DEPENDENCY_SOURCE=BUNDLED +``` + +### Use Only System Libraries (fail if not found) + +```bash +cmake -B build -DPAIMON_DEPENDENCY_SOURCE=SYSTEM +``` + +## Configuration Options + +### Global Dependency Source Control + +The `PAIMON_DEPENDENCY_SOURCE` option controls how all dependencies are acquired: + +- **AUTO** (default): Try system libraries first, fall back to bundled build +- **BUNDLED**: Always download and build from source +- **SYSTEM**: Use only system libraries (CMake will fail if not found) +- **CONDA**: Use libraries from `$CONDA_PREFIX` environment +- **VCPKG**: Use vcpkg package manager (experimental) +- **BREW**: Use Homebrew on macOS (experimental) + +Example: +```bash +cmake -B build -DPAIMON_DEPENDENCY_SOURCE=AUTO +``` + +### Per-Dependency Source Control + +You can override individual dependencies using `_SOURCE`: + +```bash +cmake -B build \ + -DPAIMON_DEPENDENCY_SOURCE=AUTO \ + -DSnappy_SOURCE=SYSTEM \ + -Dzstd_SOURCE=BUNDLED \ + -Dlz4_SOURCE=AUTO +``` + +Supported packages: `Snappy`, `zstd`, `lz4`, `zlib`, `glog`, `fmt`, `RapidJSON`, `Arrow`, `tbb`, `Protobuf`, `avro`, `orc` + +### Specifying Library Locations + +Use `_ROOT` to specify the installation directory of a dependency: + +```bash +cmake -B build \ + -DPAIMON_DEPENDENCY_SOURCE=SYSTEM \ + -DSnappy_ROOT=/usr/local \ + -Dzstd_ROOT=/opt/zstd \ + -Dlz4_ROOT=/custom/lz4 +``` + +The build system will search for: +- Headers in `${_ROOT}/include` +- Libraries in `${_ROOT}/lib`, `${_ROOT}/lib64` + +### Unified Path Prefix + +Set all unspecified `_ROOT` variables at once using `PAIMON_PACKAGE_PREFIX`: + +```bash +cmake -B build \ + -DPAIMON_DEPENDENCY_SOURCE=SYSTEM \ + -DPAIMON_PACKAGE_PREFIX=/usr/local +``` + +This automatically sets: +- `Snappy_ROOT=/usr/local` +- `zstd_ROOT=/usr/local` +- `lz4_ROOT=/usr/local` +- etc. + +### Shared vs Static Libraries + +Control whether to link against shared or static versions of dependencies: + +```bash +# Global control +cmake -B build -DPAIMON_DEPENDENCY_USE_SHARED=OFF + +# Per-dependency control +cmake -B build \ + -DPAIMON_DEPENDENCY_USE_SHARED=OFF \ + -DPAIMON_SNAPPY_USE_SHARED=ON \ + -DPAIMON_ZSTD_USE_SHARED=OFF +``` + +Supported options: +- `PAIMON_DEPENDENCY_USE_SHARED` (default: ON) +- `PAIMON_SNAPPY_USE_SHARED` +- `PAIMON_ZSTD_USE_SHARED` +- `PAIMON_LZ4_USE_SHARED` +- `PAIMON_ZLIB_USE_SHARED` +- `PAIMON_GLOG_USE_SHARED` + +## Common Scenarios + +### Development with System Libraries + +```bash +# Install dependencies via package manager +apt-get install libsnappy-dev libzstd-dev liblz4-dev libgoogle-glog-dev + +# Build with system libraries +cmake -B build -DPAIMON_DEPENDENCY_SOURCE=AUTO +make -C build -j$(nproc) +``` + +### Conda Environment + +```bash +# Activate conda environment with dependencies +conda activate myenv +conda install -c conda-forge snappy zstd lz4-c glog arrow-cpp + +# Build using conda libraries +cmake -B build -DPAIMON_DEPENDENCY_SOURCE=CONDA +make -C build -j$(nproc) +``` + +### Custom Library Paths + +```bash +cmake -B build \ + -DPAIMON_DEPENDENCY_SOURCE=SYSTEM \ + -DSnappy_ROOT=/opt/custom/snappy \ + -Dzstd_ROOT=/opt/custom/zstd \ + -DArrow_ROOT=/opt/custom/arrow +make -C build -j$(nproc) +``` + +### Mixed Approach + +```bash +# Use system Arrow but build compression libraries +cmake -B build \ + -DPAIMON_DEPENDENCY_SOURCE=AUTO \ + -DArrow_SOURCE=SYSTEM \ + -DArrow_ROOT=/usr/local \ + -DSnappy_SOURCE=BUNDLED \ + -Dzstd_SOURCE=BUNDLED +``` + +### CI/CD with Cached Dependencies + +```bash +# First build: cache dependencies in /opt/deps +cmake -B build \ + -DPAIMON_DEPENDENCY_SOURCE=BUNDLED \ + -DCMAKE_INSTALL_PREFIX=/opt/deps +make -C build -j$(nproc) +make -C build install + +# Subsequent builds: reuse cached dependencies +cmake -B build2 \ + -DPAIMON_DEPENDENCY_SOURCE=SYSTEM \ + -DPAIMON_PACKAGE_PREFIX=/opt/deps +make -C build2 -j$(nproc) +``` + +## Troubleshooting + +### "Could not find " + +If you get an error like "Could not find Snappy": + +1. Check that the package is installed: + ```bash + # Ubuntu/Debian + dpkg -l | grep libsnappy + + # macOS + brew list | grep snappy + ``` + +2. Specify the installation path: + ```bash + cmake -B build -DSnappy_ROOT=/path/to/snappy + ``` + +3. Or fall back to bundled build: + ```bash + cmake -B build -DSnappy_SOURCE=BUNDLED + ``` + +### Version Mismatch + +If the system library version is too old, you can: + +1. Force bundled build for that specific dependency: + ```bash + cmake -B build -Dzstd_SOURCE=BUNDLED + ``` + +2. Or install a newer version and specify its path: + ```bash + cmake -B build -Dzstd_ROOT=/usr/local/zstd-1.5.7 + ``` + +### Verbose Build Output + +To see detailed third-party build logs: + +```bash +cmake -B build -DPAIMON_VERBOSE_THIRDPARTY_BUILD=ON +``` + +## Implementation Status + +Currently supported dependencies with flexible resolution: +- ✅ Snappy (compression) +- ✅ zstd (compression) +- ✅ lz4 (compression) +- ✅ glog (logging) +- 🔄 zlib, fmt, RapidJSON, Arrow, tbb (basic support) +- 🔄 Protobuf, avro, orc (basic support) + +Dependencies still using BUNDLED approach: +- Jindo SDK (precompiled package) + +Legend: +- ✅ Full support with Find modules +- 🔄 Basic support (will use system if found by CMake) +- ⏳ Planned + +## See Also + +- [Issue #103](https://github.com/alibaba/paimon-cpp/issues/103) - Original feature request +- [CMake find_package documentation](https://cmake.org/cmake/help/latest/command/find_package.html) +- [Apache Arrow dependency management](https://github.com/apache/arrow/blob/main/cpp/cmake_modules/ThirdpartyToolchain.cmake)