diff --git a/README.md b/README.md index e6e8b22..7c0b5de 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,13 @@ curl https://raw.githubusercontent.com/JustinTimperio/warp-cli/master/build/inst - Fedora 32, 31, 30, 29 and 28 Workstation and Server - CentOS 8 +** OSX Catalina install (likely other MACs). +```# from terminal (with developer tools, but NOT Xcode installled) +git clone REPO +cd REPO +./build/install_osx.sh +``` + ### Uninstall Warp-CLI will remove itself from the machine but WDT will remain installed. ``` diff --git a/build/CMakeLists.txt.lts16 b/build/CMakeLists.txt.lts16 new file mode 100644 index 0000000..ede1b2b --- /dev/null +++ b/build/CMakeLists.txt.lts16 @@ -0,0 +1,448 @@ +# CMake build file - use CMake 3.2 or later to build WDT and its dependencies. +# +# mkdir build; cd build; cmake .. -DBUILD_TESTING=on; make -j +# omit -DBUILD_TESTING=on if you don't want the extra dependencies for +# testing (but testing is good !) +# (at fb: +# cd local; mkdir wdt_build; cd wdt_build +# cmake31 ~/fbcode/wdt -DFOLLY_SOURCE_DIR=$HOME/fbcode -DBUILD_TESTING=on +# make -j +# ) +# +# Copyright (c) 2014-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# + +cmake_minimum_required(VERSION 3.2) + +# There is no C per se in WDT but if you use CXX only here many checks fail +# Version is Major.Minor.YYMMDDX for up to 10 releases per day (X from 0 to 9) +# Minor currently is also the protocol version - has to match with Protocol.cpp +project("WDT" LANGUAGES C CXX VERSION 1.32.1910230) + +# On MacOS this requires the latest (master) CMake (and/or CMake 3.1.1/3.2) +# WDT itself works fine with C++11 (gcc 4.8 for instance) but more recent folly +# started to require C++14, so you can either get an older snapshot of folly +# and set "11" below or use this: +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED on) + +# somehow 'option' for this doesn't seeem to work/I don't know how to make it +set(BUILD_SHARED_LIBS on CACHE Bool "build shared libs") + +# CMake default behavior should be to set rpath when needed (non system install) +# it's not so let's set this for now: +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") + + +set(CMAKE_HOST_SYSTEM "Linux-4.4.38-minit") +set(CMAKE_HOST_SYSTEM_NAME "Linux") +set(CMAKE_HOST_SYSTEM_VERSION "4.4.38-minit") +set(CMAKE_HOST_SYSTEM_PROCESSOR "aarch64") +set(CMAKE_SYSTEM "Linux-4.4.38-minit") +set(CMAKE_SYSTEM_NAME "Linux") +set(CMAKE_SYSTEM_VERSION "4.4.38-minit") +set(CMAKE_SYSTEM_PROCESSOR "aarch64") + +set(CMAKE_CROSSCOMPILING FALSE) +set(CMAKE_SYSTEM_LOADED 1) + +# Optimized by default +# TODO: This doesn't seem to work / sets default to "" instead of Release... +# set(CMAKE_BUILD_TYPE Release CACHE String "build type") +# So hardcoding for now: +#set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_BUILD_TYPE Release) + +#set(CMAKE_CXX_FLAGS "-msse4.2 -mpclmul -Wextra -Wsign-compare -Wunused-variable") +#set(CMAKE_CXX_FLAGS "-msse4.2 -mpclmul -Wextra -Wsign-compare -Wunused-variable -Wconversion -Wsign-conversion") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "_bin/wdt") + +# Check that we have the Folly source tree +set(FOLLY_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../folly" CACHE path + "Folly source tree (folly/Conv.h should be reachable from there") +# Check for folly - TODO: this doesn't work well for relative paths +# (because of relative to build dir vs relative to source tree for -I) +if(NOT EXISTS "${FOLLY_SOURCE_DIR}/folly/Conv.h") + MESSAGE(FATAL_ERROR "${FOLLY_SOURCE_DIR}/folly/Conv.h not found +Fix using: +(in a sister directory of the wdt source tree - same level:) +git clone https://github.com/facebook/folly.git +or change FOLLY_SOURCE_DIR (use ccmake or -DFOLLY_SOURCE_DIR=...) +") +endif() + + +# The part of folly that isn't pure .h and we use: +set (FOLLY_CPP_SRC +"${FOLLY_SOURCE_DIR}/folly/Conv.cpp" +"${FOLLY_SOURCE_DIR}/folly/Demangle.cpp" +"${FOLLY_SOURCE_DIR}/folly/hash/Checksum.cpp" +"${FOLLY_SOURCE_DIR}/folly/hash/detail/ChecksumDetail.cpp" +"${FOLLY_SOURCE_DIR}/folly/hash/detail/Crc32cDetail.cpp" +"${FOLLY_SOURCE_DIR}/folly/hash/detail/Crc32CombineDetail.cpp" +"${FOLLY_SOURCE_DIR}/folly/lang/ColdClass.cpp" +"${FOLLY_SOURCE_DIR}/folly/ScopeGuard.cpp" +"${FOLLY_SOURCE_DIR}/folly/lang/CString.cpp" +) + +# WDT's library proper - comes from: ls -1 *.cpp | grep -iv test +add_library(wdt_min +util/WdtSocket.cpp +util/ClientSocket.cpp +util/EncryptionUtils.cpp +util/DirectorySourceQueue.cpp +ErrorCodes.cpp +util/FileByteSource.cpp +util/FileCreator.cpp +Protocol.cpp +WdtThread.cpp +util/ThreadsController.cpp +ReceiverThread.cpp +Receiver.cpp +WdtTransferRequest.cpp +Reporting.cpp +util/ThreadTransferHistory.cpp +SenderThread.cpp +Sender.cpp +util/ServerSocket.cpp +Throttler.cpp +WdtOptions.cpp +util/FileWriter.cpp +util/TransferLogManager.cpp +util/SerializationUtil.cpp +util/Stats.cpp +WdtBase.cpp +WdtResourceController.cpp +util/CommonImpl.cpp +) +# Source files that depend on gflags and provide flags -> options init +set (WDT_FLAGS_RELATED_SRC +util/WdtFlags.cpp +Wdt.cpp +) +add_library(wdt ${WDT_FLAGS_RELATED_SRC}) + +target_link_libraries(wdt wdt_min) + + + +# Folly uses boost system and filesystem +#set(Boost_USE_STATIC_LIBS on) +find_package(Boost COMPONENTS system filesystem REQUIRED) +include_directories(${Boost_INCLUDE_DIRS}) + +# We use std:: threads +find_package(Threads) # this will set ${CMAKE_THREAD_LIBS_INIT} (ie pthreads) + + +# double-conversion +find_path(DOUBLECONV_INCLUDE_DIR double-conversion/double-conversion.h) +find_library(DOUBLECONV_LIBRARY double-conversion) +# Glog +find_path(GLOG_INCLUDE_DIR glog/logging.h) +find_library(GLOG_LIBRARY glog) +# Gflags +find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h) +find_library(GFLAGS_LIBRARY gflags) +# OpenSSL's crypto lib +find_package(OpenSSL REQUIRED) +include_directories(${OPENSSL_INCLUDE_DIR}) + +# You can also add jemalloc to the list if you have it/want it +target_link_libraries(wdt_min + folly4wdt + ${GLOG_LIBRARY} + ${GFLAGS_LIBRARY} + ${Boost_LIBRARIES} + ${DOUBLECONV_LIBRARY} + ${OPENSSL_CRYPTO_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} # Must be last to avoid link errors +) + +# What we need to build the part of folly we use: + +include(CheckIncludeFileCXX) +include(CheckFunctionExists) +include(CheckLibraryExists) +include(CheckCXXSourceCompiles) +# For WDT itself: +check_function_exists(posix_fallocate HAS_POSIX_FALLOCATE) +check_function_exists(sync_file_range HAS_SYNC_FILE_RANGE) +check_function_exists(posix_memalign HAS_POSIX_MEMALIGN) +check_function_exists(posix_fadvise HAS_POSIX_FADVISE) +# C based check (which fail with the c++ setting thereafter...) +check_function_exists(clock_gettime FOLLY_HAVE_CLOCK_GETTIME) +# was: check_library_exists(rt clock_gettime "" FOLLY_HAVE_CLOCK_GETTIME) +check_function_exists(memrchr FOLLY_HAVE_MEMRCHR) +set(SAVE_CMRL ${CMAKE_REQUIRED_LIBRARIES}) #globals are evil/ugly +set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) +check_function_exists(pthread_atfork FOLLY_HAVE_PTHREAD_ATFORK) +set(CMAKE_REQUIRED_LIBRARIES ${SAVE_CMRL}) #globals are evil/ugly +# Needed until Cmake issue #15361 is addressed +# All the check_function_exists should be before this line +# or else they will not work' + +set(CMAKE_REQUIRED_DEFINITIONS ${CMAKE_CXX11_STANDARD_COMPILE_OPTION}) +check_include_file_cxx(linux/sockios.h WDT_HAS_SOCKIOS_H) +#check_function_exists(clock_gettime FOLLY_HAVE_CLOCK_GETTIME) +check_cxx_source_compiles("#include + #if !_LIBCPP_VERSION + #error No libc++ + #endif + int main() {return 0;}" FOLLY_USE_LIBCPP) +check_cxx_source_compiles( +"extern \"C\" void cmkcheckweak() __attribute__((weak)); +int main(int argc, char** argv) { + return (cmkcheckweak) ? 1 : 0 ; +}" FOLLY_HAVE_WEAK_SYMBOLS) +# Now record all this : +# Folly's: +configure_file(build/folly-config.h.in folly/folly-config.h) +# Wdt's config/version +configure_file(WdtConfig.h.in wdt/WdtConfig.h) + +# Malloc stuff tied to not supporting weaksympbols +if (NOT FOLLY_HAVE_WEAK_SYMBOLS) + list(APPEND FOLLY_CPP_SRC "${FOLLY_SOURCE_DIR}/folly/memory/detail/MallocImpl.cpp") + message(STATUS "no weak symbols, adding MallocImpl to folly src") +endif() + +add_library(folly4wdt ${FOLLY_CPP_SRC}) +target_link_libraries(folly4wdt ${GLOG_LIBRARY} ${DOUBLECONV_LIBRARY}) + +# Order is important - inside fb we want the above +# folly-config.h to be picked up instead of the fbcode one +include_directories(${CMAKE_CURRENT_BINARY_DIR}) +include_directories(${FOLLY_SOURCE_DIR}) +include_directories(${DOUBLECONV_INCLUDE_DIR}) +include_directories(${GLOG_INCLUDE_DIR}) +include_directories(${GFLAGS_INCLUDE_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..) + +add_executable(wdtbin wdtCmdLine.cpp ${WDT_FLAGS_RELATED_SRC}) +set_target_properties(wdtbin PROPERTIES COMPILE_DEFINITIONS "STANDALONE_APP") + +target_link_libraries(wdtbin wdt_min) + +### Install rules +set_target_properties(wdtbin PROPERTIES RUNTIME_OUTPUT_NAME "wdt") +install(TARGETS wdtbin wdt wdt_min folly4wdt + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + ) + +### Install header files + +# Find the . files in the root directory + +file(GLOB headers "*.h") +foreach(header ${headers}) + install(FILES ${header} DESTINATION include/wdt) +endforeach() + +# Install the .h files in the util with directory structure maintained + +install(DIRECTORY "${PROJECT_SOURCE_DIR}/util" DESTINATION include/wdt + FILES_MATCHING PATTERN "*.h") + +# wcp script +install(PROGRAMS test/wcp.sh DESTINATION bin RENAME wcp) + + +### Everything below is about testing (and not required to create wdt/wdt) + +if (BUILD_TESTING) + + enable_testing() + + # Extra code that we use in tests + add_library(wdt4tests_min + test/TestCommon.cpp + ) + + include(ExternalProject) + + # GTest + set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gtest") + externalproject_add( + gtest + GIT_REPOSITORY https://github.com/google/googletest.git + INSTALL_COMMAND "" # Disable install step + UPDATE_COMMAND "" # Doesn't change often + PREFIX "${GTEST_PREFIX}" + #CONFIGURE_COMMAND "" # skip + #BUILD_COMMAND "" # skip + LOG_DOWNLOAD ON + LOG_CONFIGURE ON + LOG_BUILD ON + ) + # Specify include dir for gtest + externalproject_get_property(gtest SOURCE_DIR) + include_directories("${SOURCE_DIR}/googletest/include") + + externalproject_get_property(gtest BINARY_DIR) + + # add_library(gmock_all STATIC EXCLUDE_FROM_ALL + # ${GMOCK_PREFIX}/src/gmock/gtest/src/gtest-all.cc + # ${GMOCK_PREFIX}/src/gmock/gmock-all.cc + # ${GMOCK_PREFIX}/src/gmock/gmock_main.cc) + + add_dependencies(wdt4tests_min gtest) + + # ${BINARY_DIR}/libgtest.a works everywhere except xcode... +# so ugly weird hack generating warnings about unknown dir for now: + target_link_libraries(wdt4tests_min + #"-L ${BINARY_DIR}/googlemock/gtest -L ${BINARY_DIR}/Debug -lgtest" + "-L ${BINARY_DIR}/googlemock/gtest -lgtest" + wdt_min + ) + + add_library(wdt4tests + ${WDT_FLAGS_RELATED_SRC} + ) + target_link_libraries(wdt4tests wdt4tests_min) + + # TODO: make a macro/function to add tests in 1 line instead of 3 + + + # WDT testing/benchmarking code + add_library(wdtbenchlib + bench/Bigram.cpp + ) + + target_link_libraries(wdtbenchlib + ${GLOG_LIBRARY} + ${GFLAGS_LIBRARY} + ) + + add_library(wdtbenchtestslib + bench/WdtGenTestUtils.cpp + ) + add_dependencies(wdtbenchtestslib gtest) + target_link_libraries(wdtbenchtestslib + "-L ${BINARY_DIR}/googlemock/gtest -lgtest" + wdtbenchlib + ${CMAKE_THREAD_LIBS_INIT} # Must be last to avoid link errors + ) + + add_executable(wdt_gen_files bench/wdtGenFiles.cpp) + target_link_libraries(wdt_gen_files wdtbenchlib + ${CMAKE_THREAD_LIBS_INIT} # Must be last to avoid link errors + ) + set_target_properties(wdt_gen_files PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "_bin/wdt/bench/") + + add_executable(wdt_gen_stats bench/wdtStats.cpp) + target_link_libraries(wdt_gen_stats wdtbenchlib) + + add_executable(wdt_gen_test bench/wdtGenTest.cpp) + target_link_libraries(wdt_gen_test wdtbenchtestslib) + add_test(NAME AllTestsInGenTest COMMAND wdt_gen_test) + + # Regular tests + + add_executable(protocol_test test/ProtocolTest.cpp) + target_link_libraries(protocol_test wdt4tests) + add_test(NAME AllTestsInProtocolTest COMMAND protocol_test) + + add_executable(test_encdeci64_func test/test_encdeci64_func.cpp) + target_link_libraries(test_encdeci64_func wdt4tests) + add_test(NAME test_encdeci64_func COMMAND test_encdeci64_func) + + add_executable(test_stats test/Stats_test.cpp) + target_link_libraries(test_stats wdt4tests) + add_test(NAME test_stats COMMAND test_stats) + + add_executable(histogram test/Histogram.cpp) + target_link_libraries(histogram wdt_min) + + add_executable(resource_controller_test test/WdtResourceControllerTest.cpp) + target_link_libraries(resource_controller_test wdt4tests) + add_test(NAME ResourceControllerTests COMMAND resource_controller_test) + + add_executable(wdt_url_test test/WdtUrlTest.cpp) + target_link_libraries(wdt_url_test wdt4tests) + add_test(NAME WdtUrlTests COMMAND wdt_url_test) + + add_executable(wdt_misc_tests test/WdtMiscTests.cpp) + target_link_libraries(wdt_misc_tests wdt4tests) + add_test(NAME WdtMiscTests COMMAND wdt_misc_tests) + + add_executable(wdt_fd_test test/FdTest.cpp) + target_link_libraries(wdt_fd_test wdt4tests) + add_test(NAME WdtFdTests COMMAND wdt_fd_test) + + add_executable(encryption_test test/EncryptionTest.cpp) + target_link_libraries(encryption_test wdt4tests) + add_test(NAME EncryptionTests COMMAND encryption_test) + + add_executable(file_reader_test test/FileReaderTest.cpp) + target_link_libraries(file_reader_test wdt4tests) + add_test(NAME FileReaderTests COMMAND file_reader_test) + + add_executable(option_type_test_long_flags test/OptionTypeTest.cpp) + target_link_libraries(option_type_test_long_flags wdt4tests) + + add_executable(option_type_test_short_flags test/OptionTypeTest.cpp + ${WDT_FLAGS_RELATED_SRC} + ) + set_target_properties(option_type_test_short_flags PROPERTIES + COMPILE_DEFINITIONS "STANDALONE_APP" + RUNTIME_OUTPUT_DIRECTORY "_bin/wdt/short_flags/") + + target_link_libraries(option_type_test_short_flags wdt4tests_min) + + add_test(NAME WdtRandGenTest COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_rand_gen_test.sh") + + add_test(NAME WdtBasicE2E COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_e2e_simple_test.sh") + +# Doesn't work on a mac: +# add_test(NAME WdtStdinManifestAndUrl COMMAND +# "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_stdin_test.sh") + + add_test(NAME WdtLockFailFast COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_lock_failfast.sh") + + add_test(NAME WdtBasicE2Exfs COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_e2e_xfs_test.sh") + + add_test(NAME WdtOptionsTypeTests COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_option_type_test.sh") + + add_test(NAME hostname_override_test COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/hostname_override_test.py") + + add_test(NAME WdtPortBlockTest COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_port_block_test.py") + + add_test(NAME WdtProtocolNegotiationTest COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_protocol_negotiation_test.py") + + add_test(NAME WdtSimpleOdirectTest COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_e2e_simple_test.sh" -o true) + + add_test(NAME WdtFileListTest COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_file_list_test.py") + + add_test(NAME WdtOverwriteTest COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_overwrite_test.py") + + add_test(NAME WdtBadServerTest COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_bad_server_test.py") + + add_test(NAME ReceiverThrottlerRefCountTest COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/receiver_throttler_ref_count_test.sh") + + add_test(NAME WdtLongRunningTest COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/test/wdt_long_running_test.py") + +endif(BUILD_TESTING) diff --git a/build/install_osx.sh b/build/install_osx.sh new file mode 100755 index 0000000..0ec1546 --- /dev/null +++ b/build/install_osx.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash + +if [[ "root" == `whoami` ]]; then + " + Hi + +It is not advised to run this whole install script as root. You will be prompted for the sudo p/w when it is needed. Please re-run w/out using sudo or as root. + +If you do still wish to run as root, simply delete this if statement and re-run...brew is going to complain though. + +" + exit 33; +fi + + +# Base Vars +base=/opt/warp-cli +bd=$base/build +bd_tmp=$bd/wdt/_build + +######################################### +# Tested on OSX Catalina 10.15.7 +####################################### + +echo '====================================' +echo ' Installing Dependencies' +echo '====================================' +echo '' + +# Check if developertools are installed. +echo """ +Checking to see if the OSX developer tools are installed. +If not, you'll get a popup asking if you'd like to install +the tools. You do. Install them, this will take ~15m. If +You have the tools installed, you'll just see your version +of git. oh- if you do install the dev tools, you won't +need to do it again unless you decide to upgrade. + +A few commands will require you to enter your sudo p/w, ie: +the final install. + +""" + +sleep 4 + +echo "Ok, checking if git is installed...." + +git --version + +echo "Well, off we go." + +sleep .5 + +echo '' +echo '====================================' +echo ' gtest needed for latter MACs ' +echo '====================================' +echo '' + +git clone https://github.com/google/googletest gtest +cd gtest +mkdir build +cd build +cmake .. -DCMAKE_CXX_STANDARD=17 +make +make install +cd ../../../ + +echo '' +echo '====================================' +echo ' BREW STUFF ' +echo '====================================' +echo '' + +brew install cmake +brew install glog gflags boost +brew install double-conversion +brew install openssl +brew install folly + + +##################### +# Install Warp-CLI +##################### + +echo '' +echo '====================================' +echo ' Installing Warp-CLI' +echo '====================================' +echo '' + +if ! test -f $base; then + sudo rm -rf $base +fi + +sudo git clone https://github.com/iamh2o/warp-cli.git $base ## <-=- Change back after testing + +# Link to /usr/bin +sudo rm /usr/local/bin/warp +sudo ln -s $base/core/warp.py /usr/local/bin/warp + +################################# +# Manual Build of WDT and Libs +############################### + +echo '' +echo '====================================' +echo ' Fetching WDT' +echo '====================================' +echo '' + +# Clone and Build WDT From Source +sudo git clone https://github.com/facebook/wdt.git $bd/wdt +sudo cd $bd +sudo mkdir $bd_tmp + +echo '' +echo ' Hack To Get Around Version Incompatability Bug ' +echo ' This should work, but has some risk to corrupting ' +echo ' the source code being manipulated. ' +echo '' + +# The integer set for the WDT_VERSION_BUIILD IS +# Too large for MAC compilers for some reason +# And throw a ld: malformed 64-bit a.b.c.d.e version number: 1.32.1910230 +# error. I'm arbitrarily setting it to luck 64, hopefully this is +# resolved in the main branch soon + +PVER=`grep -sE 'WDT_VERSION_BUILD' $bd/wdt/WdtConfig.h | cut -d " " -f 3` +echo ' DETECTED WDT_V_B: $PVER ' +CMD1="perl -pi -e 's/$PVER/64/g;' $bd/wdt/CMakeLists.txt" +echo $CMD1 +sudo chmod a+wrx $bd/wdt/ +sudo chmod a+wrx $bd/wdt/* +eval $CMD1 +CMD2="perl -pi -e 's/$PVER/64/g;' $bd/wdt/WdtConfig.h" +echo $CMD2 +eval $CMD2 + + +echo '' +echo '====================================' +echo ' Installing WDT ' +echo '====================================' +echo '' + +# Make and Install wdt +sudo cd $bd_tmp +sudo cmake $bd/wdt -G "Unix Makefiles" -DBUILD_TESTING=off -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl -DWDT_USE_SYSTEM_FOLLY=/usr/local/Cellar/folly/*/ +sudo make -j 2 + +sudo make install + +#echo '' +#echo '===============and==================' +#echo '' +warp -v diff --git a/core/warp.py b/core/warp.py index 1895c69..6599593 100755 --- a/core/warp.py +++ b/core/warp.py @@ -1,27 +1,42 @@ #!/usr/bin/env python3 + import re import os import pickle import argparse import subprocess +import resource - -def escape_bash_input(astr): - '''Uses regex subsitution to safely escape bash input.''' - return re.sub("(!| |\$|#|&|\"|\'|\(|\)|\||<|>|`|\\\|;)", r"\\\1", astr) +# Make this configurable, but for now jack it up so file limits dont kill u +resource.setrlimit(resource.RLIMIT_NOFILE, (100000, 100000)) -def build_command(typ, arguments, fix_hostname, threads, throttle_speed, report_interval, overwrite, follow_sym, custom_parms): - ''' +def escape_bash_input(astr): + """Uses regex subsitution to safely escape bash input.""" + return re.sub("(!| |\$|#|&|\"|'|\(|\)|\||<|>|`|\\\|;)", r"\\\1", astr) + + +def build_command( + typ, + arguments, + fix_hostname, + threads, + throttle_speed, + report_interval, + overwrite, + follow_sym, + custom_parms, +): + """ Builds the list of flags and configs neededed for the WDT transfer session - ''' + """ options = [ "-num_ports=" + str(threads), "-avg_mbytes_per_sec=" + str(throttle_speed), "-progress_report_interval_millis=" + str(report_interval), "-overwrite=" + str(overwrite).lower(), - "-follow_symlinks=" + str(follow_sym).lower() - ] + "-follow_symlinks=" + str(follow_sym).lower(), + ] if custom_parms: options.append(str(custom_parms)) @@ -32,101 +47,129 @@ def build_command(typ, arguments, fix_hostname, threads, throttle_speed, report_ # Apply Hostname Fix If Needed if fix_hostname: - ip_fetch = str("/bin/ip a | /bin/grep -v -E '127.0.0.1|00:00:00:00|inet6' | /bin/grep -Po '\d+.\d+.\d+.\d+\/' | /bin/rev | /bin/cut -c2- | /bin/rev") - if typ == 'ship': - ip_cmd = '/usr/bin/ssh ' + arguments[2] + ' ' + ip_fetch + ip_fetch = str( + "/bin/ip a | /bin/grep -v -E '127.0.0.1|00:00:00:00|inet6' | /bin/grep -Po '\d+.\d+.\d+.\d+\/' | /bin/rev | /bin/cut -c2- | /bin/rev" + ) + if typ == "ship": + ip_cmd = "ssh " + arguments[2] + " " + ip_fetch - elif typ == 'push': - ip_cmd = '/usr/bin/ssh ' + arguments[1] + ' ' + ip_fetch + elif typ == "push": + ip_cmd = "ssh " + arguments[1] + " " + ip_fetch - elif typ == 'fetch': + elif typ == "fetch": ip_cmd = str(ip_fetch) # Capture IP Address raw = subprocess.Popen(ip_cmd, stdout=subprocess.PIPE, shell=True) hostname = str(raw.stdout.readline().strip())[2:-1] - src_options.insert(0, str('-hostname ' + hostname)) + src_options.insert(0, str("-hostname " + hostname)) # Build the Command Options - if typ == 'ship': - cmd = (typ, arguments[0], arguments[1], arguments[2], arguments[3], ' '.join(src_options), ' '.join(recv_options)) - - elif typ == 'push': - cmd = (typ, arguments[0], arguments[1], arguments[2], ' '.join(src_options), ' '.join(recv_options)) - - elif typ == 'fetch': - cmd = (typ, arguments[0], arguments[1], arguments[2], ' '.join(src_options), ' '.join(recv_options)) + if typ == "ship": + cmd = ( + typ, + arguments[0], + arguments[1], + arguments[2], + arguments[3], + " ".join(src_options), + " ".join(recv_options), + ) + + elif typ == "push": + cmd = ( + typ, + arguments[0], + arguments[1], + arguments[2], + " ".join(src_options), + " ".join(recv_options), + ) + + elif typ == "fetch": + cmd = ( + typ, + arguments[0], + arguments[1], + arguments[2], + " ".join(src_options), + " ".join(recv_options), + ) return cmd def run_command(cmd): - ''' + """ Run a command build by build_command() - ''' - if cmd[0] == 'ship': + """ + if cmd[0] == "ship": # Build and Start Recv Session recv_path = escape_bash_input(cmd[4]) - recv_cmd = '/usr/bin/ssh ' + cmd[3] + ' /usr/bin/wdt ' + cmd[5] + ' -directory ' + recv_path + recv_cmd = "ssh " + cmd[3] + " wdt " + cmd[5] + " -directory " + recv_path run_cmd = subprocess.Popen(recv_cmd, stdout=subprocess.PIPE, shell=True) wdt_url = str(run_cmd.stdout.readline().strip())[1:] # Build and Start Sender Session send_path = escape_bash_input(cmd[2]) - send_cmd = '/usr/bin/ssh ' + cmd[1] + ' /usr/bin/wdt ' + cmd[6] + ' -directory ' + send_path + ' -' + send_cmd = ( + "ssh " + cmd[1] + " wdt " + cmd[6] + " -directory " + send_path + " -" + ) os.system("/bin/echo " + wdt_url + " | " + send_cmd) - elif cmd[0] == 'push': + elif cmd[0] == "push": # Build Command recv_path = escape_bash_input(cmd[3]) - recv_cmd = '/usr/bin/ssh ' + cmd[2] + ' /usr/bin/wdt ' + cmd[4] + ' -directory ' + recv_path + recv_cmd = "ssh " + cmd[2] + " wdt " + cmd[4] + " -directory " + recv_path send_path = escape_bash_input(cmd[1]) - send_cmd = '/usr/bin/wdt ' + cmd[5] + ' -directory ' + send_path + ' -' + send_cmd = "wdt " + cmd[5] + " -directory " + send_path + " -" # Run Command - os.system(recv_cmd + ' | ' + send_cmd) + os.system(recv_cmd + " | " + send_cmd) - elif cmd[0] == 'fetch': + elif cmd[0] == "fetch": # Build Command recv_path = escape_bash_input(cmd[3]) - recv_cmd = '/usr/bin/wdt ' + cmd[4] + ' -directory ' + recv_path + recv_cmd = "wdt " + cmd[4] + " -directory " + recv_path send_path = escape_bash_input(cmd[2]) - send_cmd = '/usr/bin/ssh ' + cmd[1] + ' /usr/bin/wdt ' + cmd[5] + ' -directory ' + send_path + ' -' + send_cmd = ( + "ssh " + cmd[1] + " wdt " + cmd[5] + " -directory " + send_path + " -" + ) # Run Command - os.system(recv_cmd + ' | ' + send_cmd) + os.system(recv_cmd + " | " + send_cmd) def store_macro(name, cmd): - ''' + """ Stores a macro in /home/user/warp/macros - ''' + """ # Store Macro macro_name = escape_bash_input(name) - storage_path = os.path.expanduser('~') + '/.warp/macros' + storage_path = os.path.expanduser("~") + "/.warp/macros" if not os.path.exists(storage_path): os.makedirs(storage_path) # Pickle Transfer Command - with open(storage_path + '/' + macro_name, 'wb') as m: + with open(storage_path + "/" + macro_name, "wb") as m: pickle.dump(cmd, m) - print('Stored Macro Successfully!') + print("Stored Macro Successfully!") def run_macro(name): - ''' + """ Load a macro pickle and run command. - ''' + """ # Read Macro Pickle macro_name = escape_bash_input(name) - storage_path = os.path.expanduser('~') + '/.warp/macros' - if not os.path.exists(storage_path + '/' + macro_name): - print('No Macro Found With The Name: ' + name) + storage_path = os.path.expanduser("~") + "/.warp/macros" + if not os.path.exists(storage_path + "/" + macro_name): + print("No Macro Found With The Name: " + name) return - with open(storage_path + '/' + macro_name, 'rb') as m: + with open(storage_path + "/" + macro_name, "rb") as m: macro = pickle.load(m) - print('Macro Loaded Successfully!') + print("Macro Loaded Successfully!") # Run Macro run_command(macro) @@ -139,36 +182,92 @@ def run_macro(name): parser = argparse.ArgumentParser(description="A simple wrapper for the WDT CLI Tool.") # Core Transfer Commands -parser.add_argument("-s", "--ship", nargs=4, metavar=('SRC_SSH_ALIAS', 'SRC_PATH', 'RECV_SSH_ALIAS', 'RECV_PATH'), - help="Warp a remote directory to another remote directory.") -parser.add_argument("-p", "--push", nargs=3, metavar=('SRC_PATH', 'RECV_SSH_ALIAS', 'RECV_PATH'), - help="Warp a local directory to a remote directory.") -parser.add_argument("-f", "--fetch", nargs=3, metavar=('SRC_SSH_ALIAS', 'SRC_PATH', 'RECV_PATH'), - help="Warp a remote directory to a local directory.") +parser.add_argument( + "-s", + "--ship", + nargs=4, + metavar=("SRC_SSH_ALIAS", "SRC_PATH", "RECV_SSH_ALIAS", "RECV_PATH"), + help="Warp a remote directory to another remote directory.", +) +parser.add_argument( + "-p", + "--push", + nargs=3, + metavar=("SRC_PATH", "RECV_SSH_ALIAS", "RECV_PATH"), + help="Warp a local directory to a remote directory.", +) +parser.add_argument( + "-f", + "--fetch", + nargs=3, + metavar=("SRC_SSH_ALIAS", "SRC_PATH", "RECV_PATH"), + help="Warp a remote directory to a local directory.", +) # Optional Arguments -parser.add_argument("-tr", "--threads", default="8", metavar='INT', - help="Set the number of threads/ports for WDT to use.") -parser.add_argument("-ri", "--report_interval", default="3000", metavar='INT', - help="Update interval in milliseconds for transfer report updates.") -parser.add_argument("-ts", "--throttle_speed", default="-1", metavar='INT', - help=" Throttle the transfer to an average mbytes per second.") -parser.add_argument("-ow", "--overwrite", action='store_true', - help="Allow the receiver to overwrite existing files in a directory.") -parser.add_argument("-sym", "--follow_sym", action='store_true', - help="Let WDT follow symlinks during transfer.") -parser.add_argument("-cp", "--custom_parms", default="", metavar="-CUSTOM_PARM value", - help="Inject any additional parameters available in `wdt --help`.") -parser.add_argument("-fh", "--fix_hostname", action='store_true', - help="Fix hostname issues by substituting a machines hostname for local ipv4 address.") +parser.add_argument( + "-tr", + "--threads", + default="8", + metavar="INT", + help="Set the number of threads/ports for WDT to use.", +) +parser.add_argument( + "-ri", + "--report_interval", + default="3000", + metavar="INT", + help="Update interval in milliseconds for transfer report updates.", +) +parser.add_argument( + "-ts", + "--throttle_speed", + default="-1", + metavar="INT", + help=" Throttle the transfer to an average mbytes per second.", +) +parser.add_argument( + "-ow", + "--overwrite", + action="store_true", + help="Allow the receiver to overwrite existing files in a directory.", +) +parser.add_argument( + "-sym", + "--follow_sym", + action="store_true", + help="Let WDT follow symlinks during transfer.", +) +parser.add_argument( + "-cp", + "--custom_parms", + default="", + metavar="-CUSTOM_PARM value", + help="Inject any additional parameters available in `wdt --help`.", +) +parser.add_argument( + "-fh", + "--fix_hostname", + action="store_true", + help="Fix hostname issues by substituting a machines hostname for local ipv4 address.", +) # Utilities -parser.add_argument("-m", "--macro", metavar='MACRO_NAME', - help="Execute a macro by name from ~/.warp/macros.") -parser.add_argument("-gm", "--gen_macro", metavar='MACRO_NAME', - help="Generate a new macro. This will overwrite any old macro's with the same name.") -parser.add_argument("-v", "--version", action='store_true', - help="List Warp-CLI, WDT, and FOLLY Version.") +parser.add_argument( + "-m", + "--macro", + metavar="MACRO_NAME", + help="Execute a macro by name from ~/.warp/macros.", +) +parser.add_argument( + "-gm", + "--gen_macro", + metavar="MACRO_NAME", + help="Generate a new macro. This will overwrite any old macro's with the same name.", +) +parser.add_argument( + "-v", "--version", action="store_true", help="List Warp-CLI & WDT Version" +) ############ @@ -178,29 +277,56 @@ def run_macro(name): args = parser.parse_args() if args.version: - print('Warp-CLI Version: 3.0.2') - os.system('/usr/bin/wdt --version | tr a-z A-Z') - os.system('/bin/echo "FOLLY Version:" `cd /opt/warp-cli/build/folly && /usr/bin/git describe`') + print("Warp-CLI Version: 3.0.2") + os.system("wdt --version | tr a-z A-Z") + if args.ship: - cmd = build_command('ship', args.ship, args.fix_hostname, args.threads, args.throttle_speed, - args.report_interval, args.overwrite, args.follow_sym, args.custom_parms) + cmd = build_command( + "ship", + args.ship, + args.fix_hostname, + args.threads, + args.throttle_speed, + args.report_interval, + args.overwrite, + args.follow_sym, + args.custom_parms, + ) if args.gen_macro: store_macro(args.gen_macro, cmd) else: run_command(cmd) elif args.push: - cmd = build_command('push', args.push, args.fix_hostname, args.threads, args.throttle_speed, - args.report_interval, args.overwrite, args.follow_sym, args.custom_parms) + cmd = build_command( + "push", + args.push, + args.fix_hostname, + args.threads, + args.throttle_speed, + args.report_interval, + args.overwrite, + args.follow_sym, + args.custom_parms, + ) if args.gen_macro: store_macro(args.gen_macro, cmd) else: run_command(cmd) elif args.fetch: - cmd = build_command('fetch', args.fetch, args.fix_hostname, args.threads, args.throttle_speed, - args.report_interval, args.overwrite, args.follow_sym, args.custom_parms) + cmd = build_command( + "fetch", + args.fetch, + args.fix_hostname, + args.threads, + args.throttle_speed, + args.report_interval, + args.overwrite, + args.follow_sym, + args.custom_parms, + ) if args.gen_macro: store_macro(args.gen_macro, cmd) else: diff --git a/core/warp_optimized.py b/core/warp_optimized.py new file mode 100755 index 0000000..12831d5 --- /dev/null +++ b/core/warp_optimized.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +import re +import os +import pickle +import argparse +import subprocess +import resource + +# Make this configurable, but for now jack it up else you'll get hard to debug errors if you go over the file limit +resource.setrlimit(resource.RLIMIT_NOFILE, (100000, 100000)) + + +def escape_bash_input(astr): + """Uses regex subsitution to safely escape bash input.""" + return re.sub("(!| |\$|#|&|\"|'|\(|\)|\||<|>|`|\\\|;)", r"\\\1", astr) + + +def build_command( + typ, + arguments, + fix_hostname, + threads, + throttle_speed, + report_interval, + overwrite, + follow_sym, + custom_parms, +): + """ + Builds the list of flags and configs neededed for the WDT transfer session + """ + options = [ + "-num_ports=" + str(threads), + "-avg_mbytes_per_sec=" + str(throttle_speed), + "-progress_report_interval_millis=" + str(report_interval), + "-overwrite=" + str(overwrite).lower(), + "-follow_symlinks=" + str(follow_sym).lower(), + ] + + if custom_parms: + options.append(str(custom_parms)) + + # Duplicate Options + src_options = list(options) + recv_options = list(options) + + # Apply Hostname Fix If Needed + if fix_hostname: + ip_fetch = str( + "/bin/ip a | /bin/grep -v -E '127.0.0.1|00:00:00:00|inet6' | /bin/grep -Po '\d+.\d+.\d+.\d+\/' | /bin/rev | /bin/cut -c2- | /bin/rev" + ) + if typ == "ship": + ip_cmd = "/usr/bin/ssh " + arguments[2] + " " + ip_fetch + + elif typ == "push": + ip_cmd = "/usr/bin/ssh " + arguments[1] + " " + ip_fetch + + elif typ == "fetch": + ip_cmd = str(ip_fetch) + + # Capture IP Address + raw = subprocess.Popen(ip_cmd, stdout=subprocess.PIPE, shell=True) + hostname = str(raw.stdout.readline().strip())[2:-1] + src_options.insert(0, str("-hostname " + hostname)) + + # Build the Command Options + if typ == "ship": + cmd = ( + typ, + arguments[0], + arguments[1], + arguments[2], + arguments[3], + " ".join(src_options), + " ".join(recv_options), + ) + + elif typ == "push": + cmd = ( + typ, + arguments[0], + arguments[1], + arguments[2], + " ".join(src_options), + " ".join(recv_options), + ) + + elif typ == "fetch": + cmd = ( + typ, + arguments[0], + arguments[1], + arguments[2], + " ".join(src_options), + " ".join(recv_options), + ) + + return cmd + + +def run_command(cmd): + """ + Run a command build by build_command() + """ + if cmd[0] == "ship": + # Build and Start Recv Session + recv_path = escape_bash_input(cmd[4]) + recv_cmd = ( + "/usr/bin/ssh " + + cmd[3] + + " /usr/bin/wdt " + + cmd[5] + + " -directory " + + recv_path + ) + run_cmd = subprocess.Popen(recv_cmd, stdout=subprocess.PIPE, shell=True) + wdt_url = str(run_cmd.stdout.readline().strip())[1:] + + # Build and Start Sender Session + send_path = escape_bash_input(cmd[2]) + send_cmd = ( + "/usr/bin/ssh " + + cmd[1] + + " /usr/bin/wdt " + + cmd[6] + + " -directory " + + send_path + + " -" + ) + os.system("/bin/echo " + wdt_url + " | " + send_cmd) + + elif cmd[0] == "push": + # Build Command + recv_path = escape_bash_input(cmd[3]) + recv_cmd = ( + "/usr/bin/ssh " + + cmd[2] + + " /usr/bin/wdt " + + cmd[4] + + " -abort_check_interval_millis 0 -encryption_tag_interval_bytes 0 -disk_sync_interval_mb -1 -disable_preallocation -buffer_size 20048 -encryption_type none -namespace_receiver_limit 0 -max_mbytes_per_sec -1 -block_size_mbytes 6 -iv_change_interval_mb 0 -write_timeout_millis 10000 -skip_fadvise -disable_sender_verification_during_resumption -directory " + + recv_path + ) + + send_path = escape_bash_input(cmd[1]) + send_cmd = ( + "/usr/bin/wdt " + + cmd[5] + + " -encryption_tag_interval_bytes 0 -disk_sync_interval_mb -1 -disable_preallocation -buffer_size 20048 -abort_check_interval_millis 0 -block_size_mbytes 6 -odirect_reads -skip_fadvise -iv_change_interval_mb 0 -full_reporting -namespace_receiver_limit 0 -write_timeout_millis 10000 -two_phases -open_files_during_discovery -1 -disable_sender_verification_during_resumption -encryption_type none -max_mbytes_per_sec -1 -directory " + + send_path + + " -" + ) + + # Run Command + os.system(recv_cmd + " | " + send_cmd) + + elif cmd[0] == "fetch": + # Build Command + recv_path = escape_bash_input(cmd[3]) + recv_cmd = "/usr/bin/wdt " + cmd[4] + " -directory " + recv_path + send_path = escape_bash_input(cmd[2]) + send_cmd = ( + "/usr/bin/ssh " + + cmd[1] + + " /usr/bin/wdt " + + cmd[5] + + " -directory " + + send_path + + " -" + ) + + # Run Command + os.system(recv_cmd + " | " + send_cmd) + + +def store_macro(name, cmd): + """ + Stores a macro in /home/user/warp/macros + """ + # Store Macro + macro_name = escape_bash_input(name) + storage_path = os.path.expanduser("~") + "/.warp/macros" + if not os.path.exists(storage_path): + os.makedirs(storage_path) + + # Pickle Transfer Command + with open(storage_path + "/" + macro_name, "wb") as m: + pickle.dump(cmd, m) + print("Stored Macro Successfully!") + + +def run_macro(name): + """ + Load a macro pickle and run command. + """ + # Read Macro Pickle + macro_name = escape_bash_input(name) + storage_path = os.path.expanduser("~") + "/.warp/macros" + if not os.path.exists(storage_path + "/" + macro_name): + print("No Macro Found With The Name: " + name) + return + + with open(storage_path + "/" + macro_name, "rb") as m: + macro = pickle.load(m) + print("Macro Loaded Successfully!") + + # Run Macro + run_command(macro) + + +############ +# Argument Parser +######## + +parser = argparse.ArgumentParser(description="A simple wrapper for the WDT CLI Tool.") + +# Core Transfer Commands +parser.add_argument( + "-s", + "--ship", + nargs=4, + metavar=("SRC_SSH_ALIAS", "SRC_PATH", "RECV_SSH_ALIAS", "RECV_PATH"), + help="Warp a remote directory to another remote directory.", +) +parser.add_argument( + "-p", + "--push", + nargs=3, + metavar=("SRC_PATH", "RECV_SSH_ALIAS", "RECV_PATH"), + help="Warp a local directory to a remote directory.", +) +parser.add_argument( + "-f", + "--fetch", + nargs=3, + metavar=("SRC_SSH_ALIAS", "SRC_PATH", "RECV_PATH"), + help="Warp a remote directory to a local directory.", +) + +# Optional Arguments +parser.add_argument( + "-tr", + "--threads", + default="8", + metavar="INT", + help="Set the number of threads/ports for WDT to use.", +) +parser.add_argument( + "-ri", + "--report_interval", + default="3000", + metavar="INT", + help="Update interval in milliseconds for transfer report updates.", +) +parser.add_argument( + "-ts", + "--throttle_speed", + default="-1", + metavar="INT", + help=" Throttle the transfer to an average mbytes per second.", +) +parser.add_argument( + "-ow", + "--overwrite", + action="store_true", + help="Allow the receiver to overwrite existing files in a directory.", +) +parser.add_argument( + "-sym", + "--follow_sym", + action="store_true", + help="Let WDT follow symlinks during transfer.", +) +parser.add_argument( + "-cp", + "--custom_parms", + default="", + metavar="-CUSTOM_PARM value", + help="Inject any additional parameters available in `wdt --help`.", +) +parser.add_argument( + "-fh", + "--fix_hostname", + action="store_true", + help="Fix hostname issues by substituting a machines hostname for local ipv4 address.", +) + +# Utilities +parser.add_argument( + "-m", + "--macro", + metavar="MACRO_NAME", + help="Execute a macro by name from ~/.warp/macros.", +) +parser.add_argument( + "-gm", + "--gen_macro", + metavar="MACRO_NAME", + help="Generate a new macro. This will overwrite any old macro's with the same name.", +) +parser.add_argument( + "-v", + "--version", + action="store_true", + help="List Warp-CLI, WDT, and FOLLY Version.", +) + + +############ +# Trigger Core Args +######## + +args = parser.parse_args() + +if args.version: + print("Warp-CLI Version: 3.0.2") + os.system("/usr/bin/wdt --version | tr a-z A-Z") + os.system( + '/bin/echo "FOLLY Version:" `cd /opt/warp-cli/build/folly && /usr/bin/git describe`' + ) + +if args.ship: + cmd = build_command( + "ship", + args.ship, + args.fix_hostname, + args.threads, + args.throttle_speed, + args.report_interval, + args.overwrite, + args.follow_sym, + args.custom_parms, + ) + if args.gen_macro: + store_macro(args.gen_macro, cmd) + else: + run_command(cmd) + +elif args.push: + cmd = build_command( + "push", + args.push, + args.fix_hostname, + args.threads, + args.throttle_speed, + args.report_interval, + args.overwrite, + args.follow_sym, + args.custom_parms, + ) + if args.gen_macro: + store_macro(args.gen_macro, cmd) + else: + run_command(cmd) + +elif args.fetch: + cmd = build_command( + "fetch", + args.fetch, + args.fix_hostname, + args.threads, + args.throttle_speed, + args.report_interval, + args.overwrite, + args.follow_sym, + args.custom_parms, + ) + if args.gen_macro: + store_macro(args.gen_macro, cmd) + else: + run_command(cmd) + +elif args.macro: + run_macro(args.macro)