diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 3ca7818ecad2..000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: '{branch}.{build}' -skip_tags: true -image: Visual Studio 2019 -configuration: Release -platform: x64 -clone_depth: 5 -environment: - PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' - PYTHONUTF8: 1 - QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/qt51211x64_static_vs2019_16101/Qt5.12.11_x64_static_vs2019_16101.zip' - QT_DOWNLOAD_HASH: 'cf1b58107fadbf0d9a957d14dab16cde6b6eb6936a1908472da1f967dda34a3a' - QT_LOCAL_PATH: 'C:\Qt5.12.11_x64_static_vs2019_16101' - VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f' -install: -# Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. -# - cmd: pip install zmq -# The powershell block below is to set up vcpkg to install the c++ dependencies. The pseudo code is: -# a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, -# b. Append a setting to the vcpkg cmake config file to only do release builds of dependencies (skipping deubg builds saves ~5 mins). -# Note originally this block also installed the dependencies using 'vcpkg install'. Dependencies are now installed -# as part of the msbuild command using vcpkg mainfests. -- ps: | - cd c:\tools\vcpkg - $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. - git -c advice.detachedHead=false checkout $env:VCPKG_TAG - .\bootstrap-vcpkg.bat > $null - Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" - cd "$env:APPVEYOR_BUILD_FOLDER" -before_build: -# Powershell block below is to download and extract the Qt static libraries. The pseudo code is: -# a. Download the zip file with the prebuilt Qt static libraries. -# b. Check that the downloaded file matches the expected hash. -# c. Extract the zip file to the specific destination path expected by the msbuild projects. -- ps: | - Write-Host "Downloading Qt binaries."; - Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; - Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; - if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { - Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; - Write-Host "Qt binary download matched the expected hash."; - } - else { - Write-Host "ERROR: Qt binary download did not match the expected hash."; - Exit-AppveyorBuild; - } -- cmd: python build_msvc\msvc-autogen.py -build_script: -- cmd: msbuild /p:TrackFileAccess=false build_msvc\bitcoin.sln /m /v:q /nologo -after_build: -#- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe -test_script: -- cmd: src\test_bitcoin.exe -l test_suite -- cmd: src\bench_bitcoin.exe > NUL -- ps: python test\util\bitcoin-util-test.py -- cmd: python test\util\rpcauth-test.py -# Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted. -# functional tests disabled for now. See -# https://github.com/bitcoin/bitcoin/pull/18626#issuecomment-613396202 -# https://github.com/bitcoin/bitcoin/issues/18623 -# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation -artifacts: -#- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip -deploy: off diff --git a/.cirrus.yml b/.cirrus.yml index 26bd27754f10..b3ac6d06cbfd 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,9 +1,8 @@ -### Global defaults - -env: +env: # Global defaults PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" - MAKEJOBS: "-j4" + MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache + CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling process and setting this variable avoids killing the CI script itself on error CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine @@ -18,57 +17,52 @@ persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE persistent_worker: {} # https://cirrus-ci.org/guide/persistent-workers/ # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks -base_template: &BASE_TEMPLATE +filter_template: &FILTER_TEMPLATE skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == "" # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution + stateful: false # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks + +base_template: &BASE_TEMPLATE + << : *FILTER_TEMPLATE merge_base_script: - - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + # Unconditionally install git (used in fingerprint_script). - bash -c "$PACKAGE_MANAGER_INSTALL git" - - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - - git merge FETCH_HEAD # Merge base to detect silent merge conflicts - stateful: false # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks + - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + - git fetch $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge" + - git checkout FETCH_HEAD # Use merged changes to detect silent merge conflicts + +main_template: &MAIN_TEMPLATE + timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out + ccache_cache: + folder: "/tmp/ccache_dir" + ci_script: + - ./ci/test_run_all.sh global_task_template: &GLOBAL_TASK_TEMPLATE << : *BASE_TEMPLATE - timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out container: # https://cirrus-ci.org/faq/#are-there-any-limits # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel cpu: 2 + greedy: true memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers - ccache_cache: - folder: "/tmp/ccache_dir" depends_built_cache: folder: "depends/built" - ci_script: - - ./ci/test_run_all.sh + fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-list -1 HEAD ./depends) + << : *MAIN_TEMPLATE -depends_sdk_cache_template: &DEPENDS_SDK_CACHE_TEMPLATE - depends_sdk_cache: - folder: "depends/sdk-sources" +macos_native_task_template: &MACOS_NATIVE_TASK_TEMPLATE + << : *BASE_TEMPLATE + check_clang_script: + - clang --version + brew_install_script: + - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt + << : *MAIN_TEMPLATE compute_credits_template: &CREDITS_TEMPLATE # https://cirrus-ci.org/pricing/#compute-credits # Only use credits for pull requests to the main repo use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != "" -#task: -# name: "Windows" -# windows_container: -# image: cirrusci/windowsservercore:2019 -# env: -# CIRRUS_SHELL: powershell -# PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%' -# PYTHONUTF8: 1 -# QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' -# QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' -# QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' -# VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' -# VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' -# install_script: -# - choco install python --version=3.7.7 -y - task: name: 'lint [bionic]' << : *BASE_TEMPLATE @@ -84,19 +78,131 @@ task: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: 'ARM [unit tests, no functional tests] [buster]' + name: 'tidy [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: debian:buster + image: ubuntu:jammy + cpu: 2 + memory: 5G + # For faster CI feedback, immediately schedule the linters + << : *CREDITS_TEMPLATE + env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh" + +task: + name: "Win64 native [vs2022]" + << : *FILTER_TEMPLATE + windows_container: + cpu: 6 + memory: 12G + image: cirrusci/windowsservercore:visualstudio2022 + timeout_in: 120m + env: + PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%' + PYTHONUTF8: 1 + CI_VCPKG_TAG: '2022.09.27' + VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' + VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' + CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' + WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat' + QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip' + QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.5.zip' + QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.5' + QTBASEDIR: 'C:\Qt_static' + x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' + QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml' + IgnoreWarnIntDirInTempDetected: 'true' + merge_script: + - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; } + msvc_qt_built_cache: + folder: "%QTBASEDIR%" + reupload_on_changes: false + fingerprint_script: + - echo %QT_DOWNLOAD_URL% %QT_CONFIGURE_COMMAND% + - msbuild -version + populate_script: + - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip + - mkdir C:\jom + - tar -xf C:\jom.zip -C C:\jom + - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL% + - tar -xf %QT_LOCAL_PATH% -C C:\ + - '%x64_NATIVE_TOOLS%' + - cd %QT_SOURCE_DIR% + - mkdir build + - cd build + - '%QT_CONFIGURE_COMMAND% -prefix %QTBASEDIR%' + - jom + - jom install + vcpkg_tools_cache: + folder: '%VCPKG_DOWNLOADS%\tools' + reupload_on_changes: false + fingerprint_script: + - echo %CI_VCPKG_TAG% + - msbuild -version + vcpkg_binary_cache: + folder: '%VCPKG_DEFAULT_BINARY_CACHE%' + reupload_on_changes: true + fingerprint_script: + - echo %CI_VCPKG_TAG% + - type build_msvc\vcpkg.json + - msbuild -version + populate_script: + - mkdir %VCPKG_DEFAULT_BINARY_CACHE% + ccache_cache: + folder: '%CCACHE_DIR%' + install_tools_script: + - choco install --yes --no-progress ccache --version=4.6.1 + - choco install --yes --no-progress python3 --version=3.9.6 + - pip install zmq + - ccache --version + - python -VV + install_vcpkg_script: + - cd .. + - git clone --quiet https://github.com/microsoft/vcpkg.git + - cd vcpkg + - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% + - .\bootstrap-vcpkg -disableMetrics + - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake + - .\vcpkg integrate install + - .\vcpkg version + build_script: + - '%x64_NATIVE_TOOLS%' + - cd %CIRRUS_WORKING_DIR% + - ccache --zero-stats --max-size=%CCACHE_SIZE% + - python build_msvc\msvc-autogen.py + - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL%;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + - ccache --show-stats + check_script: + - src\test_bitcoin.exe -l test_suite + - src\bench_bitcoin.exe --sanity-check > NUL + - python test\util\test_runner.py + - python test\util\rpcauth-test.py + functional_tests_script: + # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". + # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance + - netsh int ipv4 set dynamicport tcp start=1025 num=64511 + - netsh int ipv6 set dynamicport tcp start=1025 num=64511 + # Exclude feature_dbcrash for now due to timeout + - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash + +task: + name: 'ARM [unit tests, no functional tests] [bullseye]' + << : *GLOBAL_TASK_TEMPLATE + arm_container: + image: debian:bullseye + cpu: 2 + memory: 8G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_arm.sh" + QEMU_USER_CMD: "" # Disable qemu and run the test natively task: - name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]' + name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:focal + image: ubuntu:jammy env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_win64.sh" @@ -105,14 +211,14 @@ task: name: '32-bit + dash [gui] [CentOS 8]' << : *GLOBAL_TASK_TEMPLATE container: - image: centos:8 + image: quay.io/centos/centos:stream8 env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV PACKAGE_MANAGER_INSTALL: "yum install -y" FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" task: - name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]' + name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [buster]' previous_releases_cache: folder: "releases" << : *GLOBAL_TASK_TEMPLATE @@ -122,49 +228,57 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" task: - name: '[depends, sanitizers: thread (TSan), no gui] [hirsute]' + name: '[TSan, depends, gui] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:hirsute + image: ubuntu:jammy cpu: 6 # Increase CPU and Memory to avoid timeout memory: 24G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - MAKEJOBS: "-j8" FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: - name: '[depends, sanitizers: memory (MSan)] [focal]' + name: '[MSan, depends] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" + MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan task: - name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [hirsute]' + name: '[ASan + LSan + UBSan + integer, no depends, USDT] [jammy]' << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:hirsute + # We can't use a 'container' for the USDT interface tests as the CirrusCI + # containers don't have privileges to hook into bitcoind. CirrusCI uses + # Google Compute Engine instances: https://cirrus-ci.org/guide/custom-vms/ + # Images can be found here: https://cloud.google.com/compute/docs/images/os-details + compute_engine_instance: + image_project: ubuntu-os-cloud + image: family/ubuntu-2204-lts # when upgrading, check if we can drop "ADD_UNTRUSTED_BPFCC_PPA" + cpu: 4 + memory: 12G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + HOME: /root/ # Only needed for compute_engine_instance FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" + MAKEJOBS: "-j4" # Avoid excessive memory use task: - name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]' + name: '[fuzzer,address,undefined,integer, no depends] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:focal + image: ubuntu:jammy cpu: 4 # Increase CPU and memory to avoid timeout memory: 16G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - MAKEJOBS: "-j8" FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: '[multiprocess, DEBUG] [focal]' + name: '[multiprocess, i686, DEBUG] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -172,48 +286,53 @@ task: memory: 16G # The default memory is sometimes just a bit too small, so double everything env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - MAKEJOBS: "-j8" - FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh" + FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" task: - name: '[no wallet] [bionic]' + name: '[no wallet, libbitcoinkernel] [bionic]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:bionic env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" + FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" task: - name: 'macOS 10.14 [gui, no tests] [focal]' - << : *DEPENDS_SDK_CACHE_TEMPLATE - << : *GLOBAL_TASK_TEMPLATE + name: 'macOS 10.15 [gui, no tests] [focal]' + << : *BASE_TEMPLATE + macos_sdk_cache: + folder: "depends/SDKs/$MACOS_SDK" + fingerprint_key: "$MACOS_SDK" + << : *MAIN_TEMPLATE container: image: ubuntu:focal env: + MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers" << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_mac.sh" task: - name: 'macOS 11 native [gui] [no depends]' - brew_install_script: - - brew install boost libevent berkeley-db4 qt@5 miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt - << : *GLOBAL_TASK_TEMPLATE - osx_instance: + name: 'macOS 13 native arm64 [gui, sqlite only] [no depends]' + macos_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: big-sur-xcode-12.5 # https://cirrus-ci.org/guide/macOS + image: ghcr.io/cirruslabs/macos-ventura-xcode:14.1 # https://cirrus-ci.org/guide/macOS + << : *MACOS_NATIVE_TASK_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV CI_USE_APT_INSTALL: "no" PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do - FILE_ENV: "./ci/test/00_setup_env_mac_host.sh" + FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh" task: name: 'ARM64 Android APK [focal]' - << : *DEPENDS_SDK_CACHE_TEMPLATE + << : *BASE_TEMPLATE + android_sdk_cache: + folder: "depends/SDKs/android" + fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.2.8568313" depends_sources_cache: folder: "depends/sources" - << : *GLOBAL_TASK_TEMPLATE + fingerprint_script: git rev-list -1 HEAD ./depends + << : *MAIN_TEMPLATE container: image: ubuntu:focal env: diff --git a/.editorconfig b/.editorconfig index 4967e675f642..ae7e92d1c8a8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,7 +13,7 @@ trim_trailing_whitespace = true [*.{h,cpp,py,sh}] indent_size = 4 -# .cirrus.yml, .appveyor.yml, .fuzzbuzz.yml, etc. +# .cirrus.yml, .fuzzbuzz.yml, etc. [*.yml] indent_size = 2 diff --git a/.gitignore b/.gitignore index b76864cc381f..6ca9d39a16b3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ src/bitcoin-gui src/bitcoin-node src/bitcoin-tx src/bitcoin-util +src/bitcoin-chainstate src/bitcoin-wallet src/test/fuzz/fuzz src/test/test_bitcoin @@ -43,8 +44,6 @@ src/obj share/setup.nsi share/qt/Info.plist -src/univalue/gen - src/qt/*.moc src/qt/moc_*.cpp src/qt/forms/ui_*.h @@ -76,13 +75,13 @@ src/qt/bitcoin-qt.includes *.log *.trs *.dmg -*.iso *.json.h *.raw.h # Only ignore unexpected patches *.patch +!contrib/guix/patches/*.patch !depends/patches/**/*.patch #libtool object files @@ -95,7 +94,6 @@ Makefile !depends/Makefile src/qt/bitcoin-qt Bitcoin-Qt.app -background.tiff* # Qt Creator Makefile.am.user @@ -148,6 +146,7 @@ db4/ osx_volname dist/ -*.background.tiff /guix-build-* + +/ci/scratch/ diff --git a/.python-version b/.python-version index 8b7b0b52e55c..cd337510bedc 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.6.12 +3.6.15 diff --git a/.tx/config b/.tx/config index 232d481e1879..20eff98d28b7 100644 --- a/.tx/config +++ b/.tx/config @@ -1,7 +1,7 @@ [main] host = https://www.transifex.com -[bitcoin.qt-translation-022x] +[o:bitcoin:p:bitcoin:r:qt-translation-024x] file_filter = src/qt/locale/bitcoin_.xlf source_file = src/qt/locale/bitcoin_en.xlf source_lang = en diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5cd4715ef0e5..0ae4ff1a923c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,10 +10,9 @@ First, in terms of structure, there is no particular concept of "Bitcoin Core developers" in the sense of privileged people. Open source often naturally revolves around a meritocracy where contributors earn trust from the developer community over time. Nevertheless, some hierarchy is necessary for practical -purposes. As such, there are repository "maintainers" who are responsible for -merging pull requests, as well as a "lead maintainer" who is responsible for the -release cycle as well as overall merging, moderation and appointment of -maintainers. +purposes. As such, there are repository maintainers who are responsible for +merging pull requests, the [release cycle](/doc/release-process.md), and +moderation. Getting Started --------------- @@ -60,8 +59,8 @@ Most communication about Bitcoin Core development happens on IRC, in the `#bitcoin-core-dev` channel on Libera Chat. The easiest way to participate on IRC is with the web client, [web.libera.chat](https://web.libera.chat/#bitcoin-core-dev). Chat history logs can be found -on [http://www.erisian.com.au/bitcoin-core-dev/](http://www.erisian.com.au/bitcoin-core-dev/) -and [http://gnusha.org/bitcoin-core-dev/](http://gnusha.org/bitcoin-core-dev/). +on [https://www.erisian.com.au/bitcoin-core-dev/](https://www.erisian.com.au/bitcoin-core-dev/) +and [https://gnusha.org/bitcoin-core-dev/](https://gnusha.org/bitcoin-core-dev/). Discussion about codebase improvements happens in GitHub issues and pull requests. @@ -81,7 +80,7 @@ facilitates social contribution, easy testing and peer review. To contribute a patch, the workflow is as follows: - 1. Fork repository ([only for the first time](https://help.github.com/en/articles/fork-a-repo)) + 1. Fork repository ([only for the first time](https://docs.github.com/en/get-started/quickstart/fork-a-repo)) 1. Create topic branch 1. Commit patches @@ -119,7 +118,7 @@ own without warnings, errors, regressions, or test failures. Commit messages should be verbose by default consisting of a short subject line (50 chars max), a blank line and detailed explanatory text as separate -paragraph(s), unless the title alone is self-explanatory (like "Corrected typo +paragraph(s), unless the title alone is self-explanatory (like "Correct typo in init.cpp") in which case a single title line is sufficient. Commit messages should be helpful to people reading your code in the future, so explain the reasoning for your decisions. Further explanation [here](https://chris.beams.io/posts/git-commit/). @@ -153,7 +152,8 @@ the pull request affects. Valid areas as: - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code - `util` or `lib` for changes to the utils or libraries - `wallet` for changes to the wallet code - - `build` for changes to the GNU Autotools or reproducible builds + - `build` for changes to the GNU Autotools or MSVC builds + - `guix` for changes to the GUIX reproducible builds Examples: @@ -182,16 +182,21 @@ for more information on helping with translations. ### Work in Progress Changes and Requests for Comments If a pull request is not to be considered for merging (yet), please -prefix the title with [WIP] or use [Tasks Lists](https://help.github.com/articles/basic-writing-and-formatting-syntax/#task-lists) +prefix the title with [WIP] or use [Tasks Lists](https://docs.github.com/en/github/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#task-lists) in the body of the pull request to indicate tasks are pending. ### Address Feedback At this stage, one should expect comments and review from other contributors. You can add more commits to your pull request by committing them locally and pushing -to your fork until you have satisfied all feedback. +to your fork. + +You are expected to reply to any review comments before your pull request is +merged. You may update the code or reject the feedback if you do not agree with +it, but you should express so in a reply. If there is outstanding feedback and +you are not actively working on it, your pull request may be closed. -Note: Code review is a burdensome but important part of the development process, and as such, certain types of pull requests are rejected. In general, if the **improvements** do not warrant the **review effort** required, the PR has a high chance of being rejected. It is up to the PR author to convince the reviewers that the changes warrant the review effort, and if reviewers are "Concept NACK'ing" the PR, the author may need to present arguments and/or do research backing their suggested changes. +Please refer to the [peer review](#peer-review) section below for more details. ### Squashing Commits @@ -211,9 +216,9 @@ Please update the resulting commit message, if needed. It should read as a coherent message. In most cases, this means not just listing the interim commits. -If you have problems with squashing or other git workflows, you can enable -"Allow edits from maintainers" in the right-hand sidebar of the GitHub web -interface and ask for help in the pull request. +If your change contains a merge commit, the above workflow may not work and you +will need to remove the merge commit first. See the next section for details on +how to rebase. Please refrain from creating several pull requests for the same change. Use the pull request that is already open (or was created earlier) to amend @@ -226,7 +231,9 @@ pull request to pull request. ### Rebasing Changes When a pull request conflicts with the target branch, you may be asked to rebase it on top of the current target branch. -The `git rebase` command will take care of rebuilding your commits on top of the new base. + + git fetch https://github.com/bitcoin/bitcoin # Fetch the latest upstream commit + git rebase FETCH_HEAD # Rebuild commits on top of the new base This project aims to have a clean git history, where code changes are only made in non-merge commits. This simplifies auditability because merge commits can be assumed to not contain arbitrary code changes. Merge commits should be signed, @@ -287,7 +294,7 @@ projects such as libsecp256k1), and is not to be confused with overall Bitcoin Network Protocol consensus changes. Whether a pull request is merged into Bitcoin Core rests with the project merge -maintainers and ultimately the project lead. +maintainers. Maintainers will take into consideration if a patch is in line with the general principles of the project; meets the minimum standards for inclusion; and will @@ -322,6 +329,14 @@ maintainers take into account the peer review when determining if there is consensus to merge a pull request (remember that discussions may have been spread out over GitHub, mailing list and IRC discussions). +Code review is a burdensome but important part of the development process, and +as such, certain types of pull requests are rejected. In general, if the +**improvements** do not warrant the **review effort** required, the PR has a +high chance of being rejected. It is up to the PR author to convince the +reviewers that the changes warrant the review effort, and if reviewers are +"Concept NACK'ing" the PR, the author may need to present arguments and/or do +research backing their suggested changes. + #### Conceptual Review A review can be a conceptual review, where the reviewer leaves a comment @@ -386,7 +401,7 @@ about: - It may be because your code is too complex for all but a few people, and those people may not have realized your pull request even exists. A great way to find people who are qualified and care about the code you are touching is the - [Git Blame feature](https://help.github.com/articles/tracing-changes-in-a-file/). Simply + [Git Blame feature](https://docs.github.com/en/github/managing-files-in-a-repository/managing-files-on-github/tracking-changes-in-a-file). Simply look up who last modified the code you are changing and see if you can find them and give them a nudge. Don't be incessant about the nudging, though. - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request @@ -422,11 +437,6 @@ https://github.com/bitcoin/bitcoin/pull/16189). Also see the [backport.py script]( https://github.com/bitcoin-core/bitcoin-maintainer-tools#backport). -Release Policy --------------- - -The project leader is the release manager for each Bitcoin Core release. - Copyright --------- diff --git a/COPYING b/COPYING index c34f575b8c6c..b157c5fe4908 100644 --- a/COPYING +++ b/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2021 The Bitcoin Core developers -Copyright (c) 2009-2021 Bitcoin Developers +Copyright (c) 2009-2022 The Bitcoin Core developers +Copyright (c) 2009-2022 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/INSTALL.md b/INSTALL.md index 520a47d96078..4cead0303612 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,5 +1 @@ -Building Bitcoin -================ - -See doc/build-*.md for instructions on building the various -elements of the Bitcoin Core reference implementation of Bitcoin. +See [doc/build-\*.md](/doc) \ No newline at end of file diff --git a/Makefile.am b/Makefile.am index 79c294fd1585..8b61763ae4d6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,7 +12,7 @@ if ENABLE_MAN SUBDIRS += doc/man endif .PHONY: deploy FORCE -.INTERMEDIATE: $(OSX_TEMP_ISO) $(COVERAGE_INFO) +.INTERMEDIATE: $(COVERAGE_INFO) export PYTHONPATH @@ -23,6 +23,7 @@ endif BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT) BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) +BITCOIN_TEST_BIN=$(top_builddir)/src/test/$(BITCOIN_TEST_NAME)$(EXEEXT) BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT) BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT) BITCOIN_UTIL_BIN=$(top_builddir)/src/$(BITCOIN_UTIL_NAME)$(EXEEXT) @@ -37,10 +38,6 @@ space := $(empty) $(empty) OSX_APP=Bitcoin-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) OSX_DMG = $(OSX_VOLNAME).dmg -OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso -OSX_BACKGROUND_SVG=background.svg -OSX_BACKGROUND_IMAGE=background.tiff -OSX_BACKGROUND_IMAGE_DPIS=36 72 OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed @@ -50,7 +47,8 @@ DIST_CONTRIB = \ $(top_srcdir)/test/sanitizer_suppressions/tsan \ $(top_srcdir)/test/sanitizer_suppressions/ubsan \ $(top_srcdir)/contrib/linearize/linearize-data.py \ - $(top_srcdir)/contrib/linearize/linearize-hashes.py + $(top_srcdir)/contrib/linearize/linearize-hashes.py \ + $(top_srcdir)/contrib/signet/miner DIST_SHARE = \ $(top_srcdir)/share/genbuild.sh \ @@ -58,8 +56,7 @@ DIST_SHARE = \ BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py \ - $(top_srcdir)/contrib/devtools/utils.py \ - $(top_srcdir)/contrib/devtools/pixie.py + $(top_srcdir)/contrib/devtools/utils.py WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/share/pixmaps/nsis-header.bmp \ @@ -67,8 +64,6 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/doc/README_windows.txt OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \ - $(top_srcdir)/contrib/macdeploy/$(OSX_BACKGROUND_SVG) \ - $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh COVERAGE_INFO = $(COV_TOOL_WRAPPER) baseline.info \ @@ -84,6 +79,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive $(MKDIR_P) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIND_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $(top_builddir)/release + STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TEST_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release @@ -128,47 +124,22 @@ osx_volname: echo $(OSX_VOLNAME) >$@ if BUILD_DARWIN -$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(OSX_BACKGROUND_IMAGE) +$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -dmg -$(OSX_BACKGROUND_IMAGE).png: contrib/macdeploy/$(OSX_BACKGROUND_SVG) - sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 36 -p 36 -o $@ -$(OSX_BACKGROUND_IMAGE)@2x.png: contrib/macdeploy/$(OSX_BACKGROUND_SVG) - sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 72 -p 72 -o $@ -$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE).png $(OSX_BACKGROUND_IMAGE)@2x.png - tiffutil -cathidpicheck $^ -out $@ - deploydir: $(OSX_DMG) else !BUILD_DARWIN APP_DIST_DIR=$(top_builddir)/dist -APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE) $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications - -$(APP_DIST_DIR)/Applications: - @rm -f $@ - @cd $(@D); $(LN_S) /Applications $(@F) - -$(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt -$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) +$(OSX_DMG): deploydir $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ $(APP_DIST_DIR) -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH)) -$(OSX_DMG): $(OSX_TEMP_ISO) - $(DMG) dmg "$<" "$@" - -dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG) - sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@ -OSX_BACKGROUND_IMAGE_DPIFILES := $(foreach dpi,$(OSX_BACKGROUND_IMAGE_DPIS),dpi$(dpi).$(OSX_BACKGROUND_IMAGE)) -$(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE_DPIFILES) - $(MKDIR_P) $(@D) - $(TIFFCP) -c none $(OSX_BACKGROUND_IMAGE_DPIFILES) $@ - $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) - INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) + INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -deploydir: $(APP_DIST_EXTRAS) +deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt endif !BUILD_DARWIN -appbundle: $(OSX_APP_BUILT) deploy: $(OSX_DMG) endif @@ -205,8 +176,8 @@ LCOV_FILTER_PATTERN = \ -p "src/leveldb/" \ -p "src/crc32c/" \ -p "src/bench/" \ - -p "src/univalue" \ -p "src/crypto/ctaes" \ + -p "src/minisketch" \ -p "src/secp256k1" \ -p "depends" @@ -286,7 +257,7 @@ EXTRA_DIST += \ test/fuzz EXTRA_DIST += \ - test/util/bitcoin-util-test.py \ + test/util/test_runner.py \ test/util/data/bitcoin-util-test.json \ test/util/data/blanktxv1.hex \ test/util/data/blanktxv1.json \ @@ -336,6 +307,7 @@ EXTRA_DIST += \ test/util/data/txcreatescript4.json \ test/util/data/txcreatescript5.hex \ test/util/data/txcreatescript6.hex \ + test/util/data/txcreatesignsegwit1.hex \ test/util/data/txcreatesignv1.hex \ test/util/data/txcreatesignv1.json \ test/util/data/txcreatesignv2.hex \ @@ -363,18 +335,18 @@ clean-docs: clean-local: clean-docs rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ - rm -rf osx_volname dist/ dpi36.background.tiff dpi72.background.tiff + rm -rf osx_volname dist/ test-security-check: if TARGET_DARWIN - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO endif if TARGET_WINDOWS - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE endif if TARGET_LINUX - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF - $(AM_V_at) CC='$(CC)' CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF endif diff --git a/README.md b/README.md index 56132d7496d4..2eab2315eb6e 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,14 @@ https://bitcoincore.org For an immediately usable, binary version of the Bitcoin Core software, see https://bitcoincore.org/en/download/. -Further information about Bitcoin Core is available in the [doc folder](/doc). - -What is Bitcoin? ----------------- +What is Bitcoin Core? +--------------------- -Bitcoin is an experimental digital currency that enables instant payments to -anyone, anywhere in the world. Bitcoin uses peer-to-peer technology to operate -with no central authority: managing transactions and issuing money are carried -out collectively by the network. Bitcoin Core is the name of open source -software which enables the use of this currency. +Bitcoin Core connects to the Bitcoin peer-to-peer network to download and fully +validate blocks and transactions. It also includes a wallet and graphical user +interface, which can be optionally built. -For more information read the original Bitcoin whitepaper. +Further information about Bitcoin Core is available in the [doc folder](/doc). License ------- diff --git a/REVIEWERS b/REVIEWERS index e048036e4236..cb1bafa49605 100644 --- a/REVIEWERS +++ b/REVIEWERS @@ -15,124 +15,3 @@ # review a pull request. Peer review is always welcome and is a critical # component of the progress of the codebase. Information on peer review # guidelines can be found in the CONTRIBUTING.md doc. - - -# Maintainers -# @fanquake -# @hebasto -# @jonasschnelli -# @laanwj -# @marcofalke -# @meshcollider -# @sipa - -# Docs -/doc/*[a-zA-Z-].md @harding -/doc/Doxyfile.in @fanquake -/doc/REST-interface.md @jonasschnelli -/doc/benchmarking.md @ariard -/doc/bitcoin-conf.md @hebasto -/doc/build-freebsd.md @fanquake -/doc/build-netbsd.md @fanquake -/doc/build-openbsd.md @laanwj -/doc/build-osx.md @fanquake -/doc/build-unix.md @laanwj -/doc/build-windows.md @sipsorcery -/doc/dependencies.md @fanquake -/doc/developer-notes.md @laanwj -/doc/files.md @hebasto -/doc/gitian-building.md @laanwj -/doc/reduce-memory.md @fanquake -/doc/reduce-traffic.md @jonasschnelli -/doc/release-process.md @laanwj -/doc/translation_strings_policy.md @laanwj - -# Build aux -/build-aux/m4/bitcoin_qt.m4 @hebasto - -# MSVC build system -/build_msvc/ @sipsorcery - -# Settings -/src/util/settings.* @ryanofsky - -# Fuzzing - -# Tests -/src/test/net_peer_eviction_tests.cpp @jonatack -/test/functional/mempool_updatefromblock.py @hebasto -/test/functional/feature_asmap.py @jonatack -/test/functional/interface_bitcoin_cli.py @jonatack - -# Backwards compatibility tests -*_compatibility.py @sjors -/test/functional/wallet_upgradewallet.py @sjors @achow101 -/test/get_previous_releases.py @sjors - -# Translations -/src/util/translation.h @hebasto - -# Dev Tools -/contrib/devtools/security-check.py @fanquake -/contrib/devtools/test-security-check.py @fanquake -/contrib/devtools/symbol-check.py @fanquake - -# Gitian/Guix -/contrib/gitian-build.py @hebasto -/contrib/guix/ @dongcarl - -# Compatibility -/src/compat/glibc_* @fanquake - -# GUI -/src/qt/forms/ @hebasto - -# Wallet -/src/wallet/ @achow101 - -# CLI -/src/bitcoin-cli.cpp @jonatack - -# Coinstats -/src/node/coinstats.* @fjahr - -# Index -/src/index/ @fjahr - -# Descriptors -*descriptor* @achow101 @sipa - -# External signer -*external_signer* @sjors -/doc/external-signer.md @sjors -*signer.py @sjors - -# Interfaces -/src/interfaces/ @ryanofsky - -# DB -/src/txdb.* @jamesob -/src/dbwrapper.* @jamesob - -# Linter -/test/lint/lint-shell.sh @hebasto - -# Bech32 -/src/bech32.* @sipa -/src/bench/bech32.* @sipa - -# PSBT -/src/psbt* @achow101 -/src/node/psbt* @achow101 -/doc/psbt.md @achow101 - -# P2P -/src/net_processing.* @sipa -/src/protocol.* @sipa - -# Consensus -/src/coins.* @sipa @jamesob -/src/script/script.* @sipa -/src/script/interpreter.* @sipa -/src/validation.* @sipa -/src/consensus/ @sipa diff --git a/SECURITY.md b/SECURITY.md index 7ed96c7cea39..c0660e704290 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -13,8 +13,8 @@ The following keys may be used to communicate sensitive information to developer | Name | Fingerprint | |------|-------------| -| Wladimir van der Laan | 71A3 B167 3540 5025 D447 E8F2 7481 0B01 2346 C9A6 | -| Jonas Schnelli | 32EE 5C4C 3FA1 5CCA DB46 ABE5 29D4 BCB6 416F 53EC | | Pieter Wuille | 133E AC17 9436 F14A 5CF1 B794 860F EB80 4E66 9320 | +| Michael Ford | E777 299F C265 DD04 7930 70EB 944D 35F9 AC3D B76A | +| Andrew Chow | 1528 1230 0785 C964 44D3 334D 1756 5732 E08E 5E41 | -You can import a key by running the following command with that individual’s fingerprint: `gpg --recv-keys ""` Ensure that you put quotes around fingerprints containing spaces. +You can import a key by running the following command with that individual’s fingerprint: `gpg --keyserver hkps://keys.openpgp.org --recv-keys ""` Ensure that you put quotes around fingerprints containing spaces. diff --git a/build-aux/m4/ax_boost_base.m4 b/build-aux/m4/ax_boost_base.m4 index 7aac53c8155f..6c944b160ff7 100644 --- a/build-aux/m4/ax_boost_base.m4 +++ b/build-aux/m4/ax_boost_base.m4 @@ -11,9 +11,9 @@ # Test for the Boost C++ libraries of a particular version (or newer) # # If no path to the installed boost library is given the macro searchs -# under /usr, /usr/local, /opt, /opt/local and /opt/homebrew and evaluates the -# $BOOST_ROOT environment variable. Further documentation is available at -# . +# under /usr, /usr/local, /opt, /opt/local and /opt/homebrew and evaluates +# the $BOOST_ROOT environment variable. Further documentation is available +# at . # # This macro calls: # @@ -33,7 +33,7 @@ # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 48 +#serial 51 # example boost program (need to pass version) m4_define([_AX_BOOST_BASE_PROGRAM], @@ -114,7 +114,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ AS_CASE([${host_cpu}], [x86_64],[libsubdirs="lib64 libx32 lib lib64"], [mips*64*],[libsubdirs="lib64 lib32 lib lib64"], - [ppc64|powerpc64|s390x|sparc64|aarch64|ppc64le|powerpc64le|riscv64],[libsubdirs="lib64 lib lib64"], + [ppc64|powerpc64|s390x|sparc64|aarch64|ppc64le|powerpc64le|riscv64|e2k],[libsubdirs="lib64 lib lib64"], [libsubdirs="lib"] ) @@ -128,7 +128,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ ) dnl first we check the system location for boost libraries - dnl this location ist chosen if boost libraries are installed with the --layout=system option + dnl this location is chosen if boost libraries are installed with the --layout=system option dnl or if you install boost with RPM AS_IF([test "x$_AX_BOOST_BASE_boost_path" != "x"],[ AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION) includes in "$_AX_BOOST_BASE_boost_path/include"]) @@ -151,7 +151,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ else search_libsubdirs="$multiarch_libsubdir $libsubdirs" fi - for _AX_BOOST_BASE_boost_path_tmp in /usr /usr/local /opt /opt/local /opt/homebrew/; do + for _AX_BOOST_BASE_boost_path_tmp in /usr /usr/local /opt /opt/local /opt/homebrew ; do if test -d "$_AX_BOOST_BASE_boost_path_tmp/include/boost" && test -r "$_AX_BOOST_BASE_boost_path_tmp/include/boost" ; then for libsubdir in $search_libsubdirs ; do if ls "$_AX_BOOST_BASE_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi diff --git a/build-aux/m4/ax_boost_filesystem.m4 b/build-aux/m4/ax_boost_filesystem.m4 deleted file mode 100644 index 12f7bc5e2ebc..000000000000 --- a/build-aux/m4/ax_boost_filesystem.m4 +++ /dev/null @@ -1,118 +0,0 @@ -# =========================================================================== -# https://www.gnu.org/software/autoconf-archive/ax_boost_filesystem.html -# =========================================================================== -# -# SYNOPSIS -# -# AX_BOOST_FILESYSTEM -# -# DESCRIPTION -# -# Test for Filesystem library from the Boost C++ libraries. The macro -# requires a preceding call to AX_BOOST_BASE. Further documentation is -# available at . -# -# This macro calls: -# -# AC_SUBST(BOOST_FILESYSTEM_LIB) -# -# And sets: -# -# HAVE_BOOST_FILESYSTEM -# -# LICENSE -# -# Copyright (c) 2009 Thomas Porschberg -# Copyright (c) 2009 Michael Tindal -# Copyright (c) 2009 Roman Rybalko -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 28 - -AC_DEFUN([AX_BOOST_FILESYSTEM], -[ - AC_ARG_WITH([boost-filesystem], - AS_HELP_STRING([--with-boost-filesystem@<:@=special-lib@:>@], - [use the Filesystem library from boost - it is possible to specify a certain library for the linker - e.g. --with-boost-filesystem=boost_filesystem-gcc-mt ]), - [ - if test "$withval" = "no"; then - want_boost="no" - elif test "$withval" = "yes"; then - want_boost="yes" - ax_boost_user_filesystem_lib="" - else - want_boost="yes" - ax_boost_user_filesystem_lib="$withval" - fi - ], - [want_boost="yes"] - ) - - if test "x$want_boost" = "xyes"; then - AC_REQUIRE([AC_PROG_CC]) - CPPFLAGS_SAVED="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - export CPPFLAGS - - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - - LIBS_SAVED=$LIBS - LIBS="$LIBS $BOOST_SYSTEM_LIB" - export LIBS - - AC_CACHE_CHECK(whether the Boost::Filesystem library is available, - ax_cv_boost_filesystem, - [AC_LANG_PUSH([C++]) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], - [[using namespace boost::filesystem; - path my_path( "foo/bar/data.txt" ); - return 0;]])], - ax_cv_boost_filesystem=yes, ax_cv_boost_filesystem=no) - AC_LANG_POP([C++]) - ]) - if test "x$ax_cv_boost_filesystem" = "xyes"; then - AC_DEFINE(HAVE_BOOST_FILESYSTEM,,[define if the Boost::Filesystem library is available]) - BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` - if test "x$ax_boost_user_filesystem_lib" = "x"; then - for libextension in `ls -r $BOOSTLIBDIR/libboost_filesystem* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break], - [link_filesystem="no"]) - done - if test "x$link_filesystem" != "xyes"; then - for libextension in `ls -r $BOOSTLIBDIR/boost_filesystem* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break], - [link_filesystem="no"]) - done - fi - else - for ax_lib in $ax_boost_user_filesystem_lib boost_filesystem-$ax_boost_user_filesystem_lib; do - AC_CHECK_LIB($ax_lib, exit, - [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break], - [link_filesystem="no"]) - done - - fi - if test "x$ax_lib" = "x"; then - AC_MSG_ERROR(Could not find a version of the Boost::Filesystem library!) - fi - if test "x$link_filesystem" != "xyes"; then - AC_MSG_ERROR(Could not link against $ax_lib !) - fi - fi - - CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - LIBS="$LIBS_SAVED" - fi -]) diff --git a/build-aux/m4/ax_boost_system.m4 b/build-aux/m4/ax_boost_system.m4 deleted file mode 100644 index 323e2a676a8e..000000000000 --- a/build-aux/m4/ax_boost_system.m4 +++ /dev/null @@ -1,121 +0,0 @@ -# =========================================================================== -# https://www.gnu.org/software/autoconf-archive/ax_boost_system.html -# =========================================================================== -# -# SYNOPSIS -# -# AX_BOOST_SYSTEM -# -# DESCRIPTION -# -# Test for System library from the Boost C++ libraries. The macro requires -# a preceding call to AX_BOOST_BASE. Further documentation is available at -# . -# -# This macro calls: -# -# AC_SUBST(BOOST_SYSTEM_LIB) -# -# And sets: -# -# HAVE_BOOST_SYSTEM -# -# LICENSE -# -# Copyright (c) 2008 Thomas Porschberg -# Copyright (c) 2008 Michael Tindal -# Copyright (c) 2008 Daniel Casimiro -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 20 - -AC_DEFUN([AX_BOOST_SYSTEM], -[ - AC_ARG_WITH([boost-system], - AS_HELP_STRING([--with-boost-system@<:@=special-lib@:>@], - [use the System library from boost - it is possible to specify a certain library for the linker - e.g. --with-boost-system=boost_system-gcc-mt ]), - [ - if test "$withval" = "no"; then - want_boost="no" - elif test "$withval" = "yes"; then - want_boost="yes" - ax_boost_user_system_lib="" - else - want_boost="yes" - ax_boost_user_system_lib="$withval" - fi - ], - [want_boost="yes"] - ) - - if test "x$want_boost" = "xyes"; then - AC_REQUIRE([AC_PROG_CC]) - AC_REQUIRE([AC_CANONICAL_BUILD]) - CPPFLAGS_SAVED="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - export CPPFLAGS - - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - - AC_CACHE_CHECK(whether the Boost::System library is available, - ax_cv_boost_system, - [AC_LANG_PUSH([C++]) - CXXFLAGS_SAVE=$CXXFLAGS - CXXFLAGS= - - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], - [[boost::system::error_category *a = 0;]])], - ax_cv_boost_system=yes, ax_cv_boost_system=no) - CXXFLAGS=$CXXFLAGS_SAVE - AC_LANG_POP([C++]) - ]) - if test "x$ax_cv_boost_system" = "xyes"; then - AC_SUBST(BOOST_CPPFLAGS) - - AC_DEFINE(HAVE_BOOST_SYSTEM,,[define if the Boost::System library is available]) - BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` - - LDFLAGS_SAVE=$LDFLAGS - if test "x$ax_boost_user_system_lib" = "x"; then - for libextension in `ls -r $BOOSTLIBDIR/libboost_system* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break], - [link_system="no"]) - done - if test "x$link_system" != "xyes"; then - for libextension in `ls -r $BOOSTLIBDIR/boost_system* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break], - [link_system="no"]) - done - fi - - else - for ax_lib in $ax_boost_user_system_lib boost_system-$ax_boost_user_system_lib; do - AC_CHECK_LIB($ax_lib, exit, - [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break], - [link_system="no"]) - done - - fi - if test "x$ax_lib" = "x"; then - AC_MSG_ERROR(Could not find a version of the Boost::System library!) - fi - if test "x$link_system" = "xno"; then - AC_MSG_ERROR(Could not link against $ax_lib !) - fi - fi - - CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - fi -]) diff --git a/build-aux/m4/ax_boost_unit_test_framework.m4 b/build-aux/m4/ax_boost_unit_test_framework.m4 deleted file mode 100644 index 4cca32fcfd65..000000000000 --- a/build-aux/m4/ax_boost_unit_test_framework.m4 +++ /dev/null @@ -1,137 +0,0 @@ -# ================================================================================= -# https://www.gnu.org/software/autoconf-archive/ax_boost_unit_test_framework.html -# ================================================================================= -# -# SYNOPSIS -# -# AX_BOOST_UNIT_TEST_FRAMEWORK -# -# DESCRIPTION -# -# Test for Unit_Test_Framework library from the Boost C++ libraries. The -# macro requires a preceding call to AX_BOOST_BASE. Further documentation -# is available at . -# -# This macro calls: -# -# AC_SUBST(BOOST_UNIT_TEST_FRAMEWORK_LIB) -# -# And sets: -# -# HAVE_BOOST_UNIT_TEST_FRAMEWORK -# -# LICENSE -# -# Copyright (c) 2008 Thomas Porschberg -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 22 - -AC_DEFUN([AX_BOOST_UNIT_TEST_FRAMEWORK], -[ - AC_ARG_WITH([boost-unit-test-framework], - AS_HELP_STRING([--with-boost-unit-test-framework@<:@=special-lib@:>@], - [use the Unit_Test_Framework library from boost - it is possible to specify a certain library for the linker - e.g. --with-boost-unit-test-framework=boost_unit_test_framework-gcc ]), - [ - if test "$withval" = "no"; then - want_boost="no" - elif test "$withval" = "yes"; then - want_boost="yes" - ax_boost_user_unit_test_framework_lib="" - else - want_boost="yes" - ax_boost_user_unit_test_framework_lib="$withval" - fi - ], - [want_boost="yes"] - ) - - if test "x$want_boost" = "xyes"; then - AC_REQUIRE([AC_PROG_CC]) - CPPFLAGS_SAVED="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - export CPPFLAGS - - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - - AC_CACHE_CHECK(whether the Boost::Unit_Test_Framework library is available, - ax_cv_boost_unit_test_framework, - [AC_LANG_PUSH([C++]) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], - [[using boost::unit_test::test_suite; - test_suite* test= BOOST_TEST_SUITE( "Unit test example 1" ); if (test == NULL) { return 1; } else { return 0; }]])], - ax_cv_boost_unit_test_framework=yes, ax_cv_boost_unit_test_framework=no) - AC_LANG_POP([C++]) - ]) - if test "x$ax_cv_boost_unit_test_framework" = "xyes"; then - AC_DEFINE(HAVE_BOOST_UNIT_TEST_FRAMEWORK,,[define if the Boost::Unit_Test_Framework library is available]) - BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` - - if test "x$ax_boost_user_unit_test_framework_lib" = "x"; then - saved_ldflags="${LDFLAGS}" - for monitor_library in `ls $BOOSTLIBDIR/libboost_unit_test_framework*.so* $BOOSTLIBDIR/libboost_unit_test_framework*.dylib* $BOOSTLIBDIR/libboost_unit_test_framework*.a* 2>/dev/null` ; do - if test -r $monitor_library ; then - libextension=`echo $monitor_library | sed 's,.*/,,' | sed -e 's;^lib\(boost_unit_test_framework.*\)\.so.*$;\1;' -e 's;^lib\(boost_unit_test_framework.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_unit_test_framework.*\)\.a.*$;\1;'` - ax_lib=${libextension} - link_unit_test_framework="yes" - else - link_unit_test_framework="no" - fi - - if test "x$link_unit_test_framework" = "xyes"; then - BOOST_UNIT_TEST_FRAMEWORK_LIB="-l$ax_lib" - AC_SUBST(BOOST_UNIT_TEST_FRAMEWORK_LIB) - break - fi - done - if test "x$link_unit_test_framework" != "xyes"; then - for libextension in `ls $BOOSTLIBDIR/boost_unit_test_framework*.dll* $BOOSTLIBDIR/boost_unit_test_framework*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_unit_test_framework.*\)\.dll.*$;\1;' -e 's;^\(boost_unit_test_framework.*\)\.a.*$;\1;'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_UNIT_TEST_FRAMEWORK_LIB="-l$ax_lib"; AC_SUBST(BOOST_UNIT_TEST_FRAMEWORK_LIB) link_unit_test_framework="yes"; break], - [link_unit_test_framework="no"]) - done - fi - else - link_unit_test_framework="no" - saved_ldflags="${LDFLAGS}" - for ax_lib in boost_unit_test_framework-$ax_boost_user_unit_test_framework_lib $ax_boost_user_unit_test_framework_lib ; do - if test "x$link_unit_test_framework" = "xyes"; then - break; - fi - for unittest_library in `ls $BOOSTLIBDIR/lib${ax_lib}.so* $BOOSTLIBDIR/lib${ax_lib}.a* 2>/dev/null` ; do - if test -r $unittest_library ; then - libextension=`echo $unittest_library | sed 's,.*/,,' | sed -e 's;^lib\(boost_unit_test_framework.*\)\.so.*$;\1;' -e 's;^lib\(boost_unit_test_framework.*\)\.a*$;\1;'` - ax_lib=${libextension} - link_unit_test_framework="yes" - else - link_unit_test_framework="no" - fi - - if test "x$link_unit_test_framework" = "xyes"; then - BOOST_UNIT_TEST_FRAMEWORK_LIB="-l$ax_lib" - AC_SUBST(BOOST_UNIT_TEST_FRAMEWORK_LIB) - break - fi - done - done - fi - if test "x$ax_lib" = "x"; then - AC_MSG_ERROR(Could not find a version of the Boost::Unit_Test_Framework library!) - fi - if test "x$link_unit_test_framework" != "xyes"; then - AC_MSG_ERROR(Could not link against $ax_lib !) - fi - fi - - CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - fi -]) diff --git a/build-aux/m4/ax_check_compile_flag.m4 b/build-aux/m4/ax_check_compile_flag.m4 index ca3639715e72..bd753b34d7dc 100644 --- a/build-aux/m4/ax_check_compile_flag.m4 +++ b/build-aux/m4/ax_check_compile_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim # Copyright (c) 2011 Maarten Bosmans # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_COMPILE_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/ax_check_link_flag.m4 b/build-aux/m4/ax_check_link_flag.m4 index eb01a6ce135e..03a30ce4c739 100644 --- a/build-aux/m4/ax_check_link_flag.m4 +++ b/build-aux/m4/ax_check_link_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim # Copyright (c) 2011 Maarten Bosmans # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_LINK_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/ax_check_preproc_flag.m4 b/build-aux/m4/ax_check_preproc_flag.m4 index ca1d5ee2b6d1..e43560fbd3b6 100644 --- a/build-aux/m4/ax_check_preproc_flag.m4 +++ b/build-aux/m4/ax_check_preproc_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim # Copyright (c) 2011 Maarten Bosmans # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_PREPROC_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 index 43087b2e6889..51a35054d08c 100644 --- a/build-aux/m4/ax_cxx_compile_stdcxx.m4 +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -10,13 +10,13 @@ # # Check for baseline language coverage in the compiler for the specified # version of the C++ standard. If necessary, add switches to CXX and -# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) -# or '14' (for the C++14 standard). +# CXXCPP to enable support. VERSION may be '11', '14', '17', or '20' for +# the respective C++ standard version. # # The second argument, if specified, indicates whether you insist on an # extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. # -std=c++11). If neither is specified, you get whatever works, with -# preference for an extended mode. +# preference for no added switch, and then for an extended mode. # # The third argument, if specified 'mandatory' or if left unspecified, # indicates that baseline support for the specified C++ standard is @@ -35,13 +35,15 @@ # Copyright (c) 2015 Moritz Klammler # Copyright (c) 2016, 2018 Krzesimir Nowak # Copyright (c) 2019 Enji Cooper +# Copyright (c) 2020 Jason Merrill +# Copyright (c) 2021 Jörn Heusipp # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 11 +#serial 14 dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro dnl (serial version number 13). @@ -50,6 +52,7 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], [$1], [14], [ax_cxx_compile_alternatives="14 1y"], [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [$1], [20], [ax_cxx_compile_alternatives="20"], [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl m4_if([$2], [], [], [$2], [ext], [], @@ -62,6 +65,16 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl AC_LANG_PUSH([C++])dnl ac_success=no + m4_if([$2], [], [dnl + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi]) + m4_if([$2], [noext], [], [dnl if test x$ac_success = xno; then for alternative in ${ax_cxx_compile_alternatives}; do @@ -140,7 +153,6 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 ) - dnl Test body for checking C++14 support m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], @@ -148,12 +160,24 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 ) +dnl Test body for checking C++17 support + m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 ) +dnl Test body for checking C++20 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_20], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_20 +) + + dnl Tests for new features in C++11 m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ @@ -949,3 +973,33 @@ namespace cxx17 #endif // __cplusplus < 201703L ]]) + + +dnl Tests for new features in C++20 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_20], [[ + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 202002L + +#error "This is not a C++20 compiler" + +#else + +#include + +namespace cxx20 +{ + +// As C++20 supports feature test macros in the standard, there is no +// immediate need to actually test for feature availability on the +// Autoconf side. + +} // namespace cxx20 + +#endif // __cplusplus < 202002L + +]]) diff --git a/build-aux/m4/ax_pthread.m4 b/build-aux/m4/ax_pthread.m4 index 1598d077ff02..9f35d139149f 100644 --- a/build-aux/m4/ax_pthread.m4 +++ b/build-aux/m4/ax_pthread.m4 @@ -14,20 +14,24 @@ # flags that are needed. (The user can also force certain compiler # flags/libs to be tested by setting these environment variables.) # -# Also sets PTHREAD_CC to any special C compiler that is needed for -# multi-threaded programs (defaults to the value of CC otherwise). (This -# is necessary on AIX to use the special cc_r compiler alias.) +# Also sets PTHREAD_CC and PTHREAD_CXX to any special C compiler that is +# needed for multi-threaded programs (defaults to the value of CC +# respectively CXX otherwise). (This is necessary on e.g. AIX to use the +# special cc_r/CC_r compiler alias.) # # NOTE: You are assumed to not only compile your program with these flags, # but also to link with them as well. For example, you might link with # $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS +# $PTHREAD_CXX $CXXFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS # # If you are only building threaded programs, you may wish to use these # variables in your default LIBS, CFLAGS, and CC: # # LIBS="$PTHREAD_LIBS $LIBS" # CFLAGS="$CFLAGS $PTHREAD_CFLAGS" +# CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS" # CC="$PTHREAD_CC" +# CXX="$PTHREAD_CXX" # # In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute constant # has a nonstandard name, this macro defines PTHREAD_CREATE_JOINABLE to @@ -83,7 +87,7 @@ # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. -#serial 27 +#serial 31 AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD]) AC_DEFUN([AX_PTHREAD], [ @@ -105,6 +109,7 @@ if test "x$PTHREAD_CFLAGS$PTHREAD_LIBS" != "x"; then ax_pthread_save_CFLAGS="$CFLAGS" ax_pthread_save_LIBS="$LIBS" AS_IF([test "x$PTHREAD_CC" != "x"], [CC="$PTHREAD_CC"]) + AS_IF([test "x$PTHREAD_CXX" != "x"], [CXX="$PTHREAD_CXX"]) CFLAGS="$CFLAGS $PTHREAD_CFLAGS" LIBS="$PTHREAD_LIBS $LIBS" AC_MSG_CHECKING([for pthread_join using $CC $PTHREAD_CFLAGS $PTHREAD_LIBS]) @@ -386,7 +391,7 @@ if test "x$ax_pthread_clang" = "xyes"; then # step ax_pthread_save_ac_link="$ac_link" ax_pthread_sed='s/conftest\.\$ac_ext/conftest.$ac_objext/g' - ax_pthread_link_step=`$as_echo "$ac_link" | sed "$ax_pthread_sed"` + ax_pthread_link_step=`AS_ECHO(["$ac_link"]) | sed "$ax_pthread_sed"` ax_pthread_2step_ac_link="($ac_compile) && (echo ==== >&5) && ($ax_pthread_link_step)" ax_pthread_save_CFLAGS="$CFLAGS" for ax_pthread_try in '' -Qunused-arguments -Wno-unused-command-line-argument unknown; do @@ -482,18 +487,28 @@ if test "x$ax_pthread_ok" = "xyes"; then [#handle absolute path differently from PATH based program lookup AS_CASE(["x$CC"], [x/*], - [AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"])], - [AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC])])]) + [ + AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"]) + AS_IF([test "x${CXX}" != "x"], [AS_IF([AS_EXECUTABLE_P([${CXX}_r])],[PTHREAD_CXX="${CXX}_r"])]) + ], + [ + AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC]) + AS_IF([test "x${CXX}" != "x"], [AC_CHECK_PROGS([PTHREAD_CXX],[${CXX}_r],[$CXX])]) + ] + ) + ]) ;; esac fi fi test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" +test -n "$PTHREAD_CXX" || PTHREAD_CXX="$CXX" AC_SUBST([PTHREAD_LIBS]) AC_SUBST([PTHREAD_CFLAGS]) AC_SUBST([PTHREAD_CC]) +AC_SUBST([PTHREAD_CXX]) # Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: if test "x$ax_pthread_ok" = "xyes"; then diff --git a/build-aux/m4/bitcoin_find_bdb48.m4 b/build-aux/m4/bitcoin_find_bdb48.m4 index 5fc5b493d36a..3ef7fab5b5ae 100644 --- a/build-aux/m4/bitcoin_find_bdb48.m4 +++ b/build-aux/m4/bitcoin_find_bdb48.m4 @@ -3,12 +3,12 @@ dnl Distributed under the MIT software license, see the accompanying dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. AC_DEFUN([BITCOIN_FIND_BDB48],[ - AC_ARG_VAR(BDB_CFLAGS, [C compiler flags for BerkeleyDB, bypasses autodetection]) - AC_ARG_VAR(BDB_LIBS, [Linker flags for BerkeleyDB, bypasses autodetection]) + AC_ARG_VAR([BDB_CFLAGS], [C compiler flags for BerkeleyDB, bypasses autodetection]) + AC_ARG_VAR([BDB_LIBS], [Linker flags for BerkeleyDB, bypasses autodetection]) - if test "x$use_bdb" = "xno"; then + if test "$use_bdb" = "no"; then use_bdb=no - elif test "x$BDB_CFLAGS" = "x"; then + elif test "$BDB_CFLAGS" = ""; then AC_MSG_CHECKING([for Berkeley DB C++ headers]) BDB_CPPFLAGS= bdbpath=X @@ -28,7 +28,7 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ #error "failed to find bdb 4.8+" #endif ]])],[ - if test "x$bdbpath" = "xX"; then + if test "$bdbpath" = "X"; then bdbpath="${searchpath}" fi ],[ @@ -45,18 +45,25 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ break ],[]) done - if test "x$bdbpath" = "xX"; then + if test "$bdbpath" = "X"; then use_bdb=no AC_MSG_RESULT([no]) - AC_MSG_ERROR([libdb_cxx headers missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) - elif test "x$bdb48path" = "xX"; then + AC_MSG_WARN([libdb_cxx headers missing]) + AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) + elif test "$bdb48path" = "X"; then BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdbpath}],db_cxx) AC_ARG_WITH([incompatible-bdb],[AS_HELP_STRING([--with-incompatible-bdb], [allow using a bdb version other than 4.8])],[ - AC_MSG_WARN([Found Berkeley DB other than 4.8; BDB wallets opened by this build will not be portable!]) + AC_MSG_WARN([Found Berkeley DB other than 4.8]) + AC_MSG_WARN([BDB (legacy) wallets opened by this build will not be portable!]) + use_bdb=yes ],[ - AC_MSG_ERROR([Found Berkeley DB other than 4.8, required for portable BDB wallets (--with-incompatible-bdb to ignore or --without-bdb to disable BDB wallet support)]) + AC_MSG_WARN([Found Berkeley DB other than 4.8]) + AC_MSG_WARN([BDB (legacy) wallets opened by this build would not be portable!]) + AC_MSG_WARN([If this is intended, pass --with-incompatible-bdb]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) + use_bdb=no ]) - use_bdb=yes else BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdb48path}],db_cxx) bdbpath="${bdb48path}" @@ -67,9 +74,9 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ fi AC_SUBST(BDB_CPPFLAGS) - if test "x$use_bdb" = "xno"; then + if test "$use_bdb" = "no"; then use_bdb=no - elif test "x$BDB_LIBS" = "x"; then + elif test "$BDB_LIBS" = ""; then # TODO: Ideally this could find the library version and make sure it matches the headers being used for searchlib in db_cxx-4.8 db_cxx db4_cxx; do AC_CHECK_LIB([$searchlib],[main],[ @@ -77,12 +84,13 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ break ]) done - if test "x$BDB_LIBS" = "x"; then - AC_MSG_ERROR([libdb_cxx missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) + if test "$BDB_LIBS" = ""; then + AC_MSG_WARN([libdb_cxx headers missing]) + AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) fi fi - if test "x$use_bdb" != "xno"; then - AC_SUBST(BDB_LIBS) + if test "$use_bdb" != "no"; then AC_DEFINE([USE_BDB], [1], [Define if BDB support should be compiled in]) use_bdb=yes fi diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 5b5a8ed16e25..a716cd9a2779 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -5,8 +5,8 @@ dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. dnl Helper for cases where a qt dependency is not met. dnl Output: If qt version is auto, set bitcoin_enable_qt to false. Else, exit. AC_DEFUN([BITCOIN_QT_FAIL],[ - if test "x$bitcoin_qt_want_version" = xauto && test "x$bitcoin_qt_force" != xyes; then - if test "x$bitcoin_enable_qt" != xno; then + if test "$bitcoin_qt_want_version" = "auto" && test "$bitcoin_qt_force" != "yes"; then + if test "$bitcoin_enable_qt" != "no"; then AC_MSG_WARN([$1; bitcoin-qt frontend will not be built]) fi bitcoin_enable_qt=no @@ -17,7 +17,7 @@ AC_DEFUN([BITCOIN_QT_FAIL],[ ]) AC_DEFUN([BITCOIN_QT_CHECK],[ - if test "x$bitcoin_enable_qt" != xno && test "x$bitcoin_qt_want_version" != xno; then + if test "$bitcoin_enable_qt" != "no" && test "$bitcoin_qt_want_version" != "no"; then true $1 else @@ -35,12 +35,12 @@ dnl Inputs: $4: If "yes", don't fail if $2 is not found. dnl Output: $1 is set to the path of $2 if found. $2 are searched in order. AC_DEFUN([BITCOIN_QT_PATH_PROGS],[ BITCOIN_QT_CHECK([ - if test "x$3" != x; then - AC_PATH_PROGS($1,$2,,$3) + if test "$3" != ""; then + AC_PATH_PROGS([$1], [$2], [], [$3]) else - AC_PATH_PROGS($1,$2) + AC_PATH_PROGS([$1], [$2]) fi - if test "x$$1" = x && test "x$4" != xyes; then + if test "$$1" = "" && test "$4" != "yes"; then BITCOIN_QT_FAIL([$1 not found]) fi ]) @@ -57,20 +57,21 @@ AC_DEFUN([BITCOIN_QT_INIT],[ [build bitcoin-qt GUI (default=auto)])], [ bitcoin_qt_want_version=$withval - if test "x$bitcoin_qt_want_version" = xyes; then + if test "$bitcoin_qt_want_version" = "yes"; then bitcoin_qt_force=yes bitcoin_qt_want_version=auto fi ], [bitcoin_qt_want_version=auto]) - AS_IF([test "x$with_gui" = xqt5_debug], + AS_IF([test "$with_gui" = "qt5_debug"], [AS_CASE([$host], [*darwin*], [qt_lib_suffix=_debug], - [*mingw*], [qt_lib_suffix=d], [qt_lib_suffix= ]); bitcoin_qt_want_version=qt5], [qt_lib_suffix= ]) + AS_CASE([$host], [*android*], [qt_lib_suffix=_$ANDROID_ARCH]) + AC_ARG_WITH([qt-incdir],[AS_HELP_STRING([--with-qt-incdir=INC_DIR],[specify qt include path (overridden by pkgconfig)])], [qt_include_path=$withval], []) AC_ARG_WITH([qt-libdir],[AS_HELP_STRING([--with-qt-libdir=LIB_DIR],[specify qt lib path (overridden by pkgconfig)])], [qt_lib_path=$withval], []) AC_ARG_WITH([qt-plugindir],[AS_HELP_STRING([--with-qt-plugindir=PLUGIN_DIR],[specify qt plugin path (overridden by pkgconfig)])], [qt_plugin_path=$withval], []) @@ -86,7 +87,7 @@ AC_DEFUN([BITCOIN_QT_INIT],[ dnl Android doesn't support D-Bus and certainly doesn't use it for notifications case $host in *android*) - if test "x$use_dbus" != xyes; then + if test "$use_dbus" != "yes"; then use_dbus=no fi ;; @@ -115,13 +116,13 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - CXXFLAGS="$PIC_FLAGS $CXXFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" + CXXFLAGS="$PIC_FLAGS $CORE_CXXFLAGS $CXXFLAGS" _BITCOIN_QT_IS_STATIC - if test "x$bitcoin_cv_static_qt" = xyes; then + if test "$bitcoin_cv_static_qt" = "yes"; then _BITCOIN_QT_CHECK_STATIC_LIBS - if test "x$qt_plugin_path" != x; then + if test "$qt_plugin_path" != ""; then if test -d "$qt_plugin_path/platforms"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms" fi @@ -136,51 +137,49 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ fi fi - AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) - if test "x$TARGET_OS" != xandroid; then + AC_DEFINE([QT_STATICPLUGIN], [1], [Define this symbol if qt plugins are static]) + if test "$TARGET_OS" != "android"; then _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMinimalIntegrationPlugin], [-lqminimal]) - AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + AC_DEFINE([QT_QPA_PLATFORM_MINIMAL], [1], [Define this symbol if the minimal qt platform exists]) fi - if test "x$TARGET_OS" = xwindows; then + if test "$TARGET_OS" = "windows"; then dnl Linking against wtsapi32 is required. See #17749 and dnl https://bugreports.qt.io/browse/QTBUG-27097. AX_CHECK_LINK_FLAG([-lwtsapi32], [QT_LIBS="$QT_LIBS -lwtsapi32"], [AC_MSG_ERROR([could not link against -lwtsapi32])]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QWindowsIntegrationPlugin], [-lqwindows]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QWindowsVistaStylePlugin], [-lqwindowsvistastyle]) - AC_DEFINE(QT_QPA_PLATFORM_WINDOWS, 1, [Define this symbol if the qt platform is windows]) - elif test "x$TARGET_OS" = xlinux; then - dnl workaround for https://bugreports.qt.io/browse/QTBUG-74874 - AX_CHECK_LINK_FLAG([-lxcb-shm], [QT_LIBS="$QT_LIBS -lxcb-shm"], [AC_MSG_ERROR([could not link against -lxcb-shm])]) + AC_DEFINE([QT_QPA_PLATFORM_WINDOWS], [1], [Define this symbol if the qt platform is windows]) + elif test "$TARGET_OS" = "linux"; then _BITCOIN_QT_CHECK_STATIC_PLUGIN([QXcbIntegrationPlugin], [-lqxcb]) - AC_DEFINE(QT_QPA_PLATFORM_XCB, 1, [Define this symbol if the qt platform is xcb]) - elif test "x$TARGET_OS" = xdarwin; then - AX_CHECK_LINK_FLAG([[-framework Carbon]],[QT_LIBS="$QT_LIBS -framework Carbon"],[AC_MSG_ERROR(could not link against Carbon framework)]) - AX_CHECK_LINK_FLAG([[-framework IOSurface]],[QT_LIBS="$QT_LIBS -framework IOSurface"],[AC_MSG_ERROR(could not link against IOSurface framework)]) - AX_CHECK_LINK_FLAG([[-framework Metal]],[QT_LIBS="$QT_LIBS -framework Metal"],[AC_MSG_ERROR(could not link against Metal framework)]) - AX_CHECK_LINK_FLAG([[-framework QuartzCore]],[QT_LIBS="$QT_LIBS -framework QuartzCore"],[AC_MSG_ERROR(could not link against QuartzCore framework)]) + AC_DEFINE([QT_QPA_PLATFORM_XCB], [1], [Define this symbol if the qt platform is xcb]) + elif test "$TARGET_OS" = "darwin"; then + AX_CHECK_LINK_FLAG([-framework Carbon], [QT_LIBS="$QT_LIBS -framework Carbon"], [AC_MSG_ERROR(could not link against Carbon framework)]) + AX_CHECK_LINK_FLAG([-framework IOSurface], [QT_LIBS="$QT_LIBS -framework IOSurface"], [AC_MSG_ERROR(could not link against IOSurface framework)]) + AX_CHECK_LINK_FLAG([-framework Metal], [QT_LIBS="$QT_LIBS -framework Metal"], [AC_MSG_ERROR(could not link against Metal framework)]) + AX_CHECK_LINK_FLAG([-framework QuartzCore], [QT_LIBS="$QT_LIBS -framework QuartzCore"], [AC_MSG_ERROR(could not link against QuartzCore framework)]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QCocoaIntegrationPlugin], [-lqcocoa]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMacStylePlugin], [-lqmacstyle]) - AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa]) - elif test "x$TARGET_OS" = xandroid; then - QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype $QT_LIBS" - AC_DEFINE(QT_QPA_PLATFORM_ANDROID, 1, [Define this symbol if the qt platform is android]) + AC_DEFINE([QT_QPA_PLATFORM_COCOA], [1], [Define this symbol if the qt platform is cocoa]) + elif test "$TARGET_OS" = "android"; then + QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lplugins_platforms_qtforandroid${qt_lib_suffix} -ljnigraphics -landroid -lqtfreetype${qt_lib_suffix} $QT_LIBS" + AC_DEFINE([QT_QPA_PLATFORM_ANDROID], [1], [Define this symbol if the qt platform is android]) fi fi CPPFLAGS=$TEMP_CPPFLAGS CXXFLAGS=$TEMP_CXXFLAGS ]) - if test "x$qt_bin_path" = x; then + if test "$qt_bin_path" = ""; then qt_bin_path="`$PKG_CONFIG --variable=host_bins ${qt_lib_prefix}Core 2>/dev/null`" fi - if test "x$use_hardening" != xno; then + if test "$use_hardening" != "no"; then BITCOIN_QT_CHECK([ - AC_MSG_CHECKING(whether -fPIE can be used with this Qt config) + AC_MSG_CHECKING([whether -fPIE can be used with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - CXXFLAGS="$PIE_FLAGS $CXXFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" + CXXFLAGS="$PIE_FLAGS $CORE_CXXFLAGS $CXXFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #ifndef QT_VERSION @@ -192,17 +191,17 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ choke #endif ]])], - [ AC_MSG_RESULT(yes); QT_PIE_FLAGS=$PIE_FLAGS ], - [ AC_MSG_RESULT(no); QT_PIE_FLAGS=$PIC_FLAGS] + [ AC_MSG_RESULT([yes]); QT_PIE_FLAGS=$PIE_FLAGS ], + [ AC_MSG_RESULT([no]); QT_PIE_FLAGS=$PIC_FLAGS] ) CPPFLAGS=$TEMP_CPPFLAGS CXXFLAGS=$TEMP_CXXFLAGS ]) else BITCOIN_QT_CHECK([ - AC_MSG_CHECKING(whether -fPIC is needed with this Qt config) + AC_MSG_CHECKING([whether -fPIC is needed with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #ifndef QT_VERSION @@ -214,8 +213,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ choke #endif ]])], - [ AC_MSG_RESULT(no)], - [ AC_MSG_RESULT(yes); QT_PIE_FLAGS=$PIC_FLAGS] + [ AC_MSG_RESULT([no])], + [ AC_MSG_RESULT([yes]); QT_PIE_FLAGS=$PIC_FLAGS] ) CPPFLAGS=$TEMP_CPPFLAGS ]) @@ -234,12 +233,12 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ MOC_DEFS="${MOC_DEFS} -DQ_OS_MAC" base_frameworks="-framework Foundation -framework AppKit" - AX_CHECK_LINK_FLAG([[$base_frameworks]],[QT_LIBS="$QT_LIBS $base_frameworks"],[AC_MSG_ERROR(could not find base frameworks)]) + AX_CHECK_LINK_FLAG([$base_frameworks], [QT_LIBS="$QT_LIBS $base_frameworks"], [AC_MSG_ERROR(could not find base frameworks)]) ]) ;; *mingw*) BITCOIN_QT_CHECK([ - AX_CHECK_LINK_FLAG([[-mwindows]],[QT_LDFLAGS="$QT_LDFLAGS -mwindows"],[AC_MSG_WARN(-mwindows linker support not detected)]) + AX_CHECK_LINK_FLAG([-mwindows], [QT_LDFLAGS="$QT_LDFLAGS -mwindows"], [AC_MSG_WARN([-mwindows linker support not detected])]) ]) esac @@ -249,26 +248,26 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ bitcoin_enable_qt=yes bitcoin_enable_qt_test=yes - if test "x$have_qt_test" = xno; then + if test "$have_qt_test" = "no"; then bitcoin_enable_qt_test=no fi bitcoin_enable_qt_dbus=no - if test "x$use_dbus" != xno && test "x$have_qt_dbus" = xyes; then + if test "$use_dbus" != "no" && test "$have_qt_dbus" = "yes"; then bitcoin_enable_qt_dbus=yes fi - if test "x$use_dbus" = xyes && test "x$have_qt_dbus" = xno; then + if test "$use_dbus" = "yes" && test "$have_qt_dbus" = "no"; then AC_MSG_ERROR([libQtDBus not found. Install libQtDBus or remove --with-qtdbus.]) fi - if test "x$LUPDATE" = x; then + if test "$LUPDATE" = ""; then AC_MSG_WARN([lupdate tool is required to update Qt translations.]) fi - if test "x$LCONVERT" = x; then + if test "$LCONVERT" = ""; then AC_MSG_WARN([lconvert tool is required to update Qt translations.]) fi ],[ bitcoin_enable_qt=no ]) - if test x$bitcoin_enable_qt = xyes; then + if test $bitcoin_enable_qt = "yes"; then AC_MSG_RESULT([$bitcoin_enable_qt ($qt_lib_prefix)]) else AC_MSG_RESULT([$bitcoin_enable_qt]) @@ -279,9 +278,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AC_SUBST(QT_LIBS) AC_SUBST(QT_LDFLAGS) AC_SUBST(QT_DBUS_INCLUDES) - AC_SUBST(QT_DBUS_LIBS) AC_SUBST(QT_TEST_INCLUDES) - AC_SUBST(QT_TEST_LIBS) AC_SUBST(QT_SELECT, qt5) AC_SUBST(MOC_DEFS) ]) @@ -349,18 +346,20 @@ AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_LIBS], [ PKG_CHECK_MODULES([QT_FB], [${qt_lib_prefix}FbSupport${qt_lib_suffix}], [QT_LIBS="$QT_FB_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_FONTDATABASE], [${qt_lib_prefix}FontDatabaseSupport${qt_lib_suffix}], [QT_LIBS="$QT_FONTDATABASE_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_THEME], [${qt_lib_prefix}ThemeSupport${qt_lib_suffix}], [QT_LIBS="$QT_THEME_LIBS $QT_LIBS"]) - if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([QT_INPUT], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_INPUT_LIBS $QT_LIBS"]) + if test "$TARGET_OS" = "linux"; then + PKG_CHECK_MODULES([QT_INPUT], [${qt_lib_prefix}InputSupport], [QT_LIBS="$QT_INPUT_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_XCBQPA], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_XCBQPA_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xdarwin; then + PKG_CHECK_MODULES([QT_XKBCOMMON], [${qt_lib_prefix}XkbCommonSupport], [QT_LIBS="$QT_XKBCOMMON_LIBS $QT_LIBS"]) + elif test "$TARGET_OS" = "darwin"; then PKG_CHECK_MODULES([QT_CLIPBOARD], [${qt_lib_prefix}ClipboardSupport${qt_lib_suffix}], [QT_LIBS="$QT_CLIPBOARD_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_GRAPHICS], [${qt_lib_prefix}GraphicsSupport${qt_lib_suffix}], [QT_LIBS="$QT_GRAPHICS_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xwindows; then + elif test "$TARGET_OS" = "windows"; then PKG_CHECK_MODULES([QT_WINDOWSUIAUTOMATION], [${qt_lib_prefix}WindowsUIAutomationSupport${qt_lib_suffix}], [QT_LIBS="$QT_WINDOWSUIAUTOMATION_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xandroid; then - PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"]) + elif test "$TARGET_OS" = "android"; then + PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport${qt_lib_suffix}], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) fi ]) @@ -391,7 +390,7 @@ AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[ BITCOIN_QT_CHECK([ PKG_CHECK_MODULES([QT_TEST], [${qt_lib_prefix}Test${qt_lib_suffix} $qt_version], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) - if test "x$use_dbus" != xno; then + if test "$use_dbus" != "no"; then PKG_CHECK_MODULES([QT_DBUS], [${qt_lib_prefix}DBus $qt_version], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no]) fi ]) diff --git a/build-aux/m4/bitcoin_runtime_lib.m4 b/build-aux/m4/bitcoin_runtime_lib.m4 new file mode 100644 index 000000000000..1a6922deca48 --- /dev/null +++ b/build-aux/m4/bitcoin_runtime_lib.m4 @@ -0,0 +1,42 @@ +# On some platforms clang builtin implementations +# require compiler-rt as a runtime library to use. +# +# See: +# - https://bugs.llvm.org/show_bug.cgi?id=28629 + +m4_define([_CHECK_RUNTIME_testbody], [[ + bool f(long long x, long long y, long long* p) + { + return __builtin_mul_overflow(x, y, p); + } + int main() { return 0; } +]]) + +AC_DEFUN([CHECK_RUNTIME_LIB], [ + + AC_LANG_PUSH([C++]) + + AC_MSG_CHECKING([for __builtin_mul_overflow]) + AC_LINK_IFELSE( + [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])], + [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow]) + ], + [ + ax_check_save_flags="$LDFLAGS" + LDFLAGS="$LDFLAGS --rtlib=compiler-rt -lgcc_s" + AC_LINK_IFELSE( + [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])], + [ + AC_MSG_RESULT([yes, with additional linker flags]) + RUNTIME_LDFLAGS="--rtlib=compiler-rt -lgcc_s" + AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow]) + ], + [AC_MSG_RESULT([no])]) + LDFLAGS="$ax_check_save_flags" + ]) + + AC_LANG_POP + AC_SUBST([RUNTIME_LDFLAGS]) +]) diff --git a/build-aux/m4/bitcoin_subdir_to_include.m4 b/build-aux/m4/bitcoin_subdir_to_include.m4 index 7841042ac877..736270afea5a 100644 --- a/build-aux/m4/bitcoin_subdir_to_include.m4 +++ b/build-aux/m4/bitcoin_subdir_to_include.m4 @@ -5,13 +5,13 @@ dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. dnl BITCOIN_SUBDIR_TO_INCLUDE([CPPFLAGS-VARIABLE-NAME],[SUBDIRECTORY-NAME],[HEADER-FILE]) dnl SUBDIRECTORY-NAME must end with a path separator AC_DEFUN([BITCOIN_SUBDIR_TO_INCLUDE],[ - if test "x$2" = "x"; then + if test "$2" = ""; then AC_MSG_RESULT([default]) else echo "#include <$2$3.h>" >conftest.cpp newinclpath=`${CXXCPP} ${CPPFLAGS} -M conftest.cpp 2>/dev/null | [ tr -d '\\n\\r\\\\' | sed -e 's/^.*[[:space:]:]\(\/[^[:space:]]*\)]$3[\.h[[:space:]].*$/\1/' -e t -e d`] AC_MSG_RESULT([${newinclpath}]) - if test "x${newinclpath}" != "x"; then + if test "${newinclpath}" != ""; then eval "$1=\"\$$1\"' -I${newinclpath}'" fi fi diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index 40639dfe618e..602b57fe4326 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -18,7 +18,7 @@ m4_define([_CHECK_ATOMIC_testbody], [[ int main() { std::atomic lock{true}; - std::atomic_exchange(&lock, false); + lock.exchange(false); std::atomic t{0s}; t.store(2s); @@ -34,6 +34,8 @@ m4_define([_CHECK_ATOMIC_testbody], [[ AC_DEFUN([CHECK_ATOMIC], [ AC_LANG_PUSH(C++) + TEMP_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS" AC_MSG_CHECKING([whether std::atomic can be used without link library]) @@ -51,5 +53,6 @@ AC_DEFUN([CHECK_ATOMIC], [ ]) ]) + CXXFLAGS="$TEMP_CXXFLAGS" AC_LANG_POP ]) diff --git a/build-aux/m4/l_filesystem.m4 b/build-aux/m4/l_filesystem.m4 new file mode 100644 index 000000000000..ca3a0cd41c06 --- /dev/null +++ b/build-aux/m4/l_filesystem.m4 @@ -0,0 +1,47 @@ +dnl Copyright (c) 2022 The Bitcoin Core developers +dnl Distributed under the MIT software license, see the accompanying +dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# GCC 8.1 and earlier requires -lstdc++fs +# Clang 8.0.0 (libc++) and earlier requires -lc++fs + +m4_define([_CHECK_FILESYSTEM_testbody], [[ + #include + + namespace fs = std::filesystem; + + int main() { + (void)fs::current_path().root_name(); + return 0; + } +]]) + +AC_DEFUN([CHECK_FILESYSTEM], [ + + AC_LANG_PUSH(C++) + + AC_MSG_CHECKING([whether std::filesystem can be used without link library]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + SAVED_LIBS="$LIBS" + LIBS="$SAVED_LIBS -lstdc++fs" + AC_MSG_CHECKING([whether std::filesystem needs -lstdc++fs]) + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + AC_MSG_CHECKING([whether std::filesystem needs -lc++fs]) + LIBS="$SAVED_LIBS -lc++fs" + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_FAILURE([cannot figure out how to use std::filesystem]) + ]) + ]) + ]) + + AC_LANG_POP +]) diff --git a/build_msvc/.gitignore b/build_msvc/.gitignore index ae8120fdf3a4..b2eb9313a063 100644 --- a/build_msvc/.gitignore +++ b/build_msvc/.gitignore @@ -13,7 +13,7 @@ packages/* libbitcoin_cli/libbitcoin_cli.vcxproj libbitcoin_common/libbitcoin_common.vcxproj libbitcoin_crypto/libbitcoin_crypto.vcxproj -libbitcoin_server/libbitcoin_server.vcxproj +libbitcoin_node/libbitcoin_node.vcxproj libbitcoin_util/libbitcoin_util.vcxproj libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj libbitcoin_wallet/libbitcoin_wallet.vcxproj @@ -21,6 +21,9 @@ libbitcoin_zmq/libbitcoin_zmq.vcxproj bench_bitcoin/bench_bitcoin.vcxproj libtest_util/libtest_util.vcxproj +/bitcoin_config.h +/common.init.vcxproj + */Win32 libbitcoin_qt/QtGeneratedFiles/* test_bitcoin-qt/QtGeneratedFiles/* diff --git a/build_msvc/README.md b/build_msvc/README.md index 88a05644a73a..7520700a3449 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -3,78 +3,69 @@ Building Bitcoin Core with Visual Studio Introduction --------------------- -Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). +Visual Studio 2022 is minimum required to build Bitcoin Core. -Building with Visual Studio is an alternative to the Linux based [cross-compiler build](https://github.com/bitcoin/bitcoin/blob/master/doc/build-windows.md). +Solution and project files to build with `msbuild` or Visual Studio can be found in the `build_msvc` directory. -Quick Start ---------------------- -The minimal steps required to build Bitcoin Core with the msbuild toolchain are below. More detailed instructions are contained in the following sections. +To build Bitcoin Core from the command-line, it is sufficient to only install the [Visual Studio Build Tools](https://visualstudio.microsoft.com/downloads/) component. -``` -cd build_msvc -py -3 msvc-autogen.py -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build -``` +The "Desktop development with C++" workload must be installed as well. -Dependencies ---------------------- -A number of [open source libraries](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) are required in order to be able to build Bitcoin Core. +Building with Visual Studio is an alternative to the Linux based [cross-compiler build](../doc/build-windows.md). -Options for installing the dependencies in a Visual Studio compatible manner are: -- Use Microsoft's [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) to download the source packages and build locally. This is the recommended approach. -- Download the source code, build each dependency, add the required include paths, link libraries and binary tools to the Visual Studio project files. -- Use [nuget](https://www.nuget.org/) packages with the understanding that any binary files have been compiled by an untrusted third party. +Prerequisites +--------------------- +To build [dependencies](../doc/dependencies.md) (except for [Qt](#qt)), +the default approach is to use the [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) package manager from Microsoft: -The [external dependencies](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) required for building are listed in the `build_msvc/vcpkg.json` file. To ensure `msbuild` project files automatically install the `vcpkg` dependencies use: +1. [Install](https://vcpkg.io/en/getting-started.html) vcpkg. -``` -vcpkg integrate install +2. By default, vcpkg makes both `release` and `debug` builds for each package. +To save build time and disk space, one could skip `debug` builds (example uses PowerShell): +```powershell + +Add-Content -Path "vcpkg\triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" ``` Qt --------------------- -In order to build Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v142) and platform type (x86 or x64) must also match. +To build Bitcoin Core with the GUI, a static build of Qt is required. + +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.5.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. + +2. Open "x64 Native Tools Command Prompt for VS 2022", and input the following commands: +```cmd +cd C:\dev\qt-source +mkdir build +cd build +..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix C:\Qt_static +nmake +nmake install +``` -Some prebuilt x64 versions of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware these downloads are NOT officially sanctioned by Bitcoin Core and are provided for developer convenience only. They should NOT be used for builds that will be used in a production environment or with real funds. +One could speed up building with [`jom`](https://wiki.qt.io/Jom), a replacement for `nmake` which makes use of all CPU cores. -To determine which Qt prebuilt version to download open the `.appveyor.yml` file and note the `QT_DOWNLOAD_URL`. When extracting the zip file the destination path must be set to `C:\`. This is due to the way that Qt includes, libraries and tools use internal paths. +To build Bitcoin Core without Qt, unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. -To build Bitcoin Core without Qt unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. Building --------------------- -The instructions below use `vcpkg` to install the dependencies. - -- Install [`vcpkg`](https://github.com/Microsoft/vcpkg). +1. Use Python to generate `*.vcxproj` for the Visual Studio 2022 toolchain from Makefile: -- Use Python to generate `*.vcxproj` from Makefile - -``` -PS >py -3 msvc-autogen.py +```cmd +python build_msvc\msvc-autogen.py ``` -- An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. +2. An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. To specify a non-default path to a static Qt package directory, use the `QTBASEDIR` environment variable. -- To build from the command line with the Visual Studio 2019 toolchain use: +3. To build from the command-line with the Visual Studio toolchain use: -``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build +```cmd +msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal ``` -- Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. - -AppVeyor ---------------------- -The .appveyor.yml in the root directory is suitable to perform builds on [AppVeyor](https://www.appveyor.com/) Continuous Integration servers. The simplest way to perform an AppVeyor build is to fork Bitcoin Core and then configure a new AppVeyor Project pointing to the forked repository. - -For safety reasons the Bitcoin Core .appveyor.yml file has the artifact options disabled. The build will be performed but no executable files will be available. To enable artifacts on a forked repository uncomment the lines shown below: - -``` - #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe - #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip -``` +Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio. Security --------------------- @@ -96,4 +87,4 @@ If is it enabled then in the output `Dynamic base` will be listed in the `DLL ch Terminal Server Aware ``` -This may not disable all stack randomization as versions of windows employ additional stack randomization protections. These protections must be turned off in the OS configuration. \ No newline at end of file +This may not disable all stack randomization as versions of windows employ additional stack randomization protections. These protections must be turned off in the OS configuration. diff --git a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in index 128c1bd8e7b6..fc9d7cbed645 100644 --- a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in +++ b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in @@ -21,7 +21,7 @@ {6190199c-6cf4-4dad-bfbd-93fa72a760c1} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} diff --git a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj index e5e0e978f898..738884fb4108 100644 --- a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj +++ b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj @@ -15,6 +15,9 @@ {0667528c-d734-4009-adf9-c0d6c4a5a5a6} + + {7c87e378-df58-482e-aa2f-1bc129bc19ce} + {6190199c-6cf4-4dad-bfbd-93fa72a760c1} diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index a697c1dfb68c..0d6358e0d077 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -9,6 +9,7 @@ + @@ -27,7 +28,7 @@ {2b4abff8-d1fd-4845-88c9-1f3c0a6512bf} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} diff --git a/build_msvc/bitcoin-util/bitcoin-util.vcxproj b/build_msvc/bitcoin-util/bitcoin-util.vcxproj index 3a6aa4a83715..8a0964824bcd 100644 --- a/build_msvc/bitcoin-util/bitcoin-util.vcxproj +++ b/build_msvc/bitcoin-util/bitcoin-util.vcxproj @@ -2,7 +2,7 @@ - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF} + {57A04EC9-542A-4E40-83D0-AC3BE1F36805} Application diff --git a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj index 40c5db552288..2ac0be9814bd 100644 --- a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj +++ b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj @@ -10,32 +10,26 @@ + + $(IntDir)init_bitcoin-wallet.obj + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} - - {0667528c-d734-4009-adf9-c0d6c4a5a5a6} - {7c87e378-df58-482e-aa2f-1bc129bc19ce} {6190199c-6cf4-4dad-bfbd-93fa72a760c1} - - {460fee33-1fe1-483f-b3bf-931ff8e969a5} - {b53a5535-ee9d-4c6f-9a26-f79ee3bc3754} {93b86837-b543-48a5-a89b-7c87abb77df2} - - {792d487f-f14c-49fc-a9de-3fc150f31c3f} - {5724ba7d-a09a-4ba8-800b-c4c1561b3d69} @@ -45,9 +39,6 @@ {bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6} - - {18430fef-6b61-4c53-b396-718e02850f1b} - diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index 7d8591c10bd0..2a1ccf58fec9 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -4,8 +4,6 @@ VisualStudioVersion = 16.0.28803.452 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoinconsensus", "libbitcoinconsensus\libbitcoinconsensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "testconsensus", "testconsensus\testconsensus.vcxproj", "{E78473E9-B850-456C-9120-276301E04C06}" -EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoind", "bitcoind\bitcoind.vcxproj", "{D4513DDF-6013-44DC-ADCC-12EAF6D1F038}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_util", "libbitcoin_util\libbitcoin_util.vcxproj", "{B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}" @@ -14,7 +12,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_common", "libbit EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_crypto", "libbitcoin_crypto\libbitcoin_crypto.vcxproj", "{6190199C-6CF4-4DAD-BFBD-93FA72A760C1}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_server", "libbitcoin_server\libbitcoin_server.vcxproj", "{460FEE33-1FE1-483F-B3BF-931FF8E969A5}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_node", "libbitcoin_node\libbitcoin_node.vcxproj", "{460FEE33-1FE1-483F-B3BF-931FF8E969A5}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libunivalue", "libunivalue\libunivalue.vcxproj", "{5724BA7D-A09A-4BA8-800B-C4C1561B3D69}" EndProject @@ -32,7 +30,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bench_bitcoin", "bench_bitc EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-tx", "bitcoin-tx\bitcoin-tx.vcxproj", "{D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-util", "bitcoin-util\bitcoin-util.vcxproj", "{D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-util", "bitcoin-util\bitcoin-util.vcxproj", "{57A04EC9-542A-4E40-83D0-AC3BE1F36805}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-wallet", "bitcoin-wallet\bitcoin-wallet.vcxproj", "{84DE8790-EDE3-4483-81AC-C32F15E861F4}" EndProject @@ -50,205 +48,115 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libtest_util", "libtest_uti EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_bitcoin-qt", "test_bitcoin-qt\test_bitcoin-qt.vcxproj", "{51201D5E-D939-4854-AE9D-008F03FF518E}" EndProject +Project("{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}") = "libminisketch", "libminisketch\libminisketch.vcxproj", "{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 Release|x64 = Release|x64 - Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x64.ActiveCfg = Debug|x64 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x64.Build.0 = Debug|x64 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x86.ActiveCfg = Debug|Win32 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x86.Build.0 = Debug|Win32 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x64.ActiveCfg = Release|x64 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x64.Build.0 = Release|x64 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x86.ActiveCfg = Release|Win32 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x86.Build.0 = Release|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x64.ActiveCfg = Debug|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x64.Build.0 = Debug|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x86.ActiveCfg = Debug|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x86.Build.0 = Debug|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x64.ActiveCfg = Release|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x64.Build.0 = Release|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x86.ActiveCfg = Release|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x86.Build.0 = Release|Win32 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x64.ActiveCfg = Debug|x64 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x64.Build.0 = Debug|x64 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x86.ActiveCfg = Debug|Win32 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x86.Build.0 = Debug|Win32 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x64.ActiveCfg = Release|x64 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x64.Build.0 = Release|x64 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x86.ActiveCfg = Release|Win32 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x86.Build.0 = Release|Win32 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x64.ActiveCfg = Debug|x64 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x64.Build.0 = Debug|x64 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x86.ActiveCfg = Debug|Win32 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x86.Build.0 = Debug|Win32 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x64.ActiveCfg = Release|x64 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x64.Build.0 = Release|x64 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x86.ActiveCfg = Release|Win32 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x86.Build.0 = Release|Win32 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x64.ActiveCfg = Debug|x64 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x64.Build.0 = Debug|x64 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x86.ActiveCfg = Debug|Win32 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x86.Build.0 = Debug|Win32 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x64.ActiveCfg = Release|x64 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x64.Build.0 = Release|x64 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x86.ActiveCfg = Release|Win32 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x86.Build.0 = Release|Win32 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x64.ActiveCfg = Debug|x64 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x64.Build.0 = Debug|x64 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x86.ActiveCfg = Debug|Win32 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x86.Build.0 = Debug|Win32 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x64.ActiveCfg = Release|x64 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x64.Build.0 = Release|x64 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x86.ActiveCfg = Release|Win32 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x86.Build.0 = Release|Win32 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x64.ActiveCfg = Debug|x64 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x64.Build.0 = Debug|x64 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x86.ActiveCfg = Debug|Win32 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x86.Build.0 = Debug|Win32 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x64.ActiveCfg = Release|x64 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x64.Build.0 = Release|x64 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x86.ActiveCfg = Release|Win32 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x86.Build.0 = Release|Win32 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x64.ActiveCfg = Debug|x64 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x64.Build.0 = Debug|x64 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x86.ActiveCfg = Debug|Win32 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x86.Build.0 = Debug|Win32 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x64.ActiveCfg = Release|x64 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x64.Build.0 = Release|x64 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x86.ActiveCfg = Release|Win32 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x86.Build.0 = Release|Win32 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x64.ActiveCfg = Debug|x64 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x64.Build.0 = Debug|x64 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x86.ActiveCfg = Debug|Win32 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x86.Build.0 = Debug|Win32 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x64.ActiveCfg = Release|x64 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x64.Build.0 = Release|x64 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x86.ActiveCfg = Release|Win32 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x86.Build.0 = Release|Win32 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x64.ActiveCfg = Debug|x64 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x64.Build.0 = Debug|x64 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x86.ActiveCfg = Debug|Win32 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x86.Build.0 = Debug|Win32 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x64.ActiveCfg = Release|x64 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x64.Build.0 = Release|x64 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x86.ActiveCfg = Release|Win32 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x86.Build.0 = Release|Win32 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.ActiveCfg = Debug|x64 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.Build.0 = Debug|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x86.ActiveCfg = Debug|Win32 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x86.Build.0 = Debug|Win32 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.ActiveCfg = Release|x64 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.Build.0 = Release|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x86.ActiveCfg = Release|Win32 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x86.Build.0 = Release|Win32 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x64.ActiveCfg = Debug|x64 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x64.Build.0 = Debug|x64 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x86.ActiveCfg = Debug|Win32 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x86.Build.0 = Debug|Win32 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x64.ActiveCfg = Release|x64 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x64.Build.0 = Release|x64 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x86.ActiveCfg = Release|Win32 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x86.Build.0 = Release|Win32 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x64.ActiveCfg = Debug|x64 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x64.Build.0 = Debug|x64 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x86.ActiveCfg = Debug|Win32 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x86.Build.0 = Debug|Win32 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x64.ActiveCfg = Release|x64 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x64.Build.0 = Release|x64 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x86.ActiveCfg = Release|Win32 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x86.Build.0 = Release|Win32 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x64.ActiveCfg = Debug|x64 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x64.Build.0 = Debug|x64 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x86.ActiveCfg = Debug|Win32 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x86.Build.0 = Debug|Win32 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x64.ActiveCfg = Release|x64 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x64.Build.0 = Release|x64 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x86.ActiveCfg = Release|Win32 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x86.Build.0 = Release|Win32 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x64.ActiveCfg = Debug|x64 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x64.Build.0 = Debug|x64 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x86.ActiveCfg = Debug|Win32 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x86.Build.0 = Debug|Win32 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x64.ActiveCfg = Release|x64 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x64.Build.0 = Release|x64 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x86.ActiveCfg = Release|Win32 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x86.Build.0 = Release|Win32 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Debug|x64.ActiveCfg = Debug|x64 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Debug|x64.Build.0 = Debug|x64 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Release|x64.ActiveCfg = Release|x64 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Release|x64.Build.0 = Release|x64 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x64.ActiveCfg = Debug|x64 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x64.Build.0 = Debug|x64 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x86.ActiveCfg = Debug|Win32 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x86.Build.0 = Debug|Win32 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x64.ActiveCfg = Release|x64 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x64.Build.0 = Release|x64 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x86.ActiveCfg = Release|Win32 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x86.Build.0 = Release|Win32 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x64.ActiveCfg = Debug|x64 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x64.Build.0 = Debug|x64 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x86.ActiveCfg = Debug|Win32 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x86.Build.0 = Debug|Win32 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x64.ActiveCfg = Release|x64 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x64.Build.0 = Release|x64 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x86.ActiveCfg = Release|Win32 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x86.Build.0 = Release|Win32 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x64.ActiveCfg = Debug|x64 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x64.Build.0 = Debug|x64 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x86.ActiveCfg = Debug|Win32 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x86.Build.0 = Debug|Win32 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x64.ActiveCfg = Release|x64 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x64.Build.0 = Release|x64 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x86.ActiveCfg = Release|Win32 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x86.Build.0 = Release|Win32 {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x64.ActiveCfg = Debug|x64 {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x64.Build.0 = Debug|x64 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x86.ActiveCfg = Debug|Win32 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x86.Build.0 = Debug|Win32 {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x64.ActiveCfg = Release|x64 {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x64.Build.0 = Release|x64 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x86.ActiveCfg = Release|Win32 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x86.Build.0 = Release|Win32 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x64.ActiveCfg = Debug|x64 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x64.Build.0 = Debug|x64 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x86.ActiveCfg = Debug|Win32 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x86.Build.0 = Debug|Win32 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x64.ActiveCfg = Release|x64 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x64.Build.0 = Release|x64 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x86.ActiveCfg = Release|Win32 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x86.Build.0 = Release|Win32 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x64.ActiveCfg = Debug|x64 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x64.Build.0 = Debug|x64 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x86.ActiveCfg = Debug|Win32 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x86.Build.0 = Debug|Win32 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x64.ActiveCfg = Release|x64 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x64.Build.0 = Release|x64 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.ActiveCfg = Release|Win32 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.Build.0 = Release|Win32 {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.ActiveCfg = Debug|x64 {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.Build.0 = Debug|x64 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.ActiveCfg = Debug|Win32 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.Build.0 = Debug|Win32 {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.ActiveCfg = Release|x64 {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.Build.0 = Release|x64 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.ActiveCfg = Release|Win32 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.Build.0 = Release|Win32 {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.ActiveCfg = Debug|x64 {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.Build.0 = Debug|x64 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.ActiveCfg = Debug|Win32 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.Build.0 = Debug|Win32 {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.ActiveCfg = Release|x64 {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.Build.0 = Release|x64 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.ActiveCfg = Release|Win32 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.Build.0 = Release|Win32 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.ActiveCfg = Debug|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.Build.0 = Debug|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.ActiveCfg = Release|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {8AA72EDA-2CD4-4564-B1E4-688B760EEEE9} - SolutionGuid = {8607C0F4-F33D-41B8-8D51-18E366A0F8DF} SolutionGuid = {58AAB032-7274-49BD-845E-5EF4DBB69B70} EndGlobalSection EndGlobal diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h deleted file mode 100644 index aba5607eb602..000000000000 --- a/build_msvc/bitcoin_config.h +++ /dev/null @@ -1,339 +0,0 @@ -// Copyright (c) 2018-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_BITCOIN_CONFIG_H -#define BITCOIN_BITCOIN_CONFIG_H - -/* Define if building universal (internal helper macro) */ -/* #undef AC_APPLE_UNIVERSAL_BUILD */ - -/* Version Build */ -#define CLIENT_VERSION_BUILD 0 - -/* Version is release */ -#define CLIENT_VERSION_IS_RELEASE false - -/* Major version */ -#define CLIENT_VERSION_MAJOR 21 - -/* Minor version */ -#define CLIENT_VERSION_MINOR 99 - -/* Copyright holder(s) before %s replacement */ -#define COPYRIGHT_HOLDERS "The %s developers" - -/* Copyright holder(s) */ -#define COPYRIGHT_HOLDERS_FINAL "The Bitcoin Core developers" - -/* Replacement for %s in copyright holders string */ -#define COPYRIGHT_HOLDERS_SUBSTITUTION "Bitcoin Core" - -/* Copyright year */ -#define COPYRIGHT_YEAR 2019 - -/* Define to 1 to enable wallet functions */ -#define ENABLE_WALLET 1 - -/* Define to 1 to enable BDB wallet */ -#define USE_BDB 1 - -/* Define to 1 to enable SQLite wallet */ -#define USE_SQLITE 1 - -/* Define to 1 to enable ZMQ functions */ -#define ENABLE_ZMQ 1 - -/* define if the Boost library is available */ -#define HAVE_BOOST /**/ - -/* define if the Boost::Filesystem library is available */ -#define HAVE_BOOST_FILESYSTEM /**/ - -/* define if external signer support is enabled (requires Boost::Process) */ -#define ENABLE_EXTERNAL_SIGNER /**/ - -/* define if the Boost::System library is available */ -#define HAVE_BOOST_SYSTEM /**/ - -/* define if the Boost::Unit_Test_Framework library is available */ -#define HAVE_BOOST_UNIT_TEST_FRAMEWORK /**/ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_BYTESWAP_H */ - -/* Define this symbol if the consensus lib has been built */ -#define HAVE_CONSENSUS_LIB 1 - -/* define if the compiler supports basic C++11 syntax */ -#define HAVE_CXX11 1 - -/* Define to 1 if you have the declaration of `be16toh', and to 0 if you - don't. */ -#define HAVE_DECL_BE16TOH 0 - -/* Define to 1 if you have the declaration of `be32toh', and to 0 if you - don't. */ -#define HAVE_DECL_BE32TOH 0 - -/* Define to 1 if you have the declaration of `be64toh', and to 0 if you - don't. */ -#define HAVE_DECL_BE64TOH 0 - -/* Define to 1 if you have the declaration of `bswap_16', and to 0 if you - don't. */ -#define HAVE_DECL_BSWAP_16 0 - -/* Define to 1 if you have the declaration of `bswap_32', and to 0 if you - don't. */ -#define HAVE_DECL_BSWAP_32 0 - -/* Define to 1 if you have the declaration of `bswap_64', and to 0 if you - don't. */ -#define HAVE_DECL_BSWAP_64 0 - -/* Define to 1 if you have the declaration of `fork', and to 0 if you don't. - */ -#define HAVE_DECL_FORK 0 - -/* Define to 1 if you have the declaration of `htobe16', and to 0 if you - don't. */ -#define HAVE_DECL_HTOBE16 0 - -/* Define to 1 if you have the declaration of `htobe32', and to 0 if you - don't. */ -#define HAVE_DECL_HTOBE32 0 - -/* Define to 1 if you have the declaration of `htobe64', and to 0 if you - don't. */ -#define HAVE_DECL_HTOBE64 0 - -/* Define to 1 if you have the declaration of `htole16', and to 0 if you - don't. */ -#define HAVE_DECL_HTOLE16 0 - -/* Define to 1 if you have the declaration of `htole32', and to 0 if you - don't. */ -#define HAVE_DECL_HTOLE32 0 - -/* Define to 1 if you have the declaration of `htole64', and to 0 if you - don't. */ -#define HAVE_DECL_HTOLE64 0 - -/* Define to 1 if you have the declaration of `le16toh', and to 0 if you - don't. */ -#define HAVE_DECL_LE16TOH 0 - -/* Define to 1 if you have the declaration of `le32toh', and to 0 if you - don't. */ -#define HAVE_DECL_LE32TOH 0 - -/* Define to 1 if you have the declaration of `le64toh', and to 0 if you - don't. */ -#define HAVE_DECL_LE64TOH 0 - -/* Define to 1 if you have the declaration of `setsid', and to 0 if you don't. - */ -#define HAVE_DECL_SETSID 0 - -/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you - don't. */ -#define HAVE_DECL_STRERROR_R 0 - -/* Define to 1 if you have the declaration of `strnlen', and to 0 if you - don't. */ -#define HAVE_DECL_STRNLEN 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_DLFCN_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_ENDIAN_H */ - -/* Define to 1 if the system has the `dllexport' function attribute */ -#define HAVE_FUNC_ATTRIBUTE_DLLEXPORT 1 - -/* Define to 1 if the system has the `dllimport' function attribute */ -#define HAVE_FUNC_ATTRIBUTE_DLLIMPORT 1 - -/* Define to 1 if the system has the `visibility' function attribute */ -#define HAVE_FUNC_ATTRIBUTE_VISIBILITY 1 - -/* Define this symbol if the BSD getentropy system call is available */ -/* #undef HAVE_GETENTROPY */ - -/* Define this symbol if the BSD getentropy system call is available with - sys/random.h */ -/* #undef HAVE_GETENTROPY_RAND */ - -/* Define to 1 if you have the header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define this symbol if you have malloc_info */ -/* #undef HAVE_MALLOC_INFO */ - -/* Define this symbol if you have mallopt with M_ARENA_MAX */ -/* #undef HAVE_MALLOPT_ARENA_MAX */ - -/* Define to 1 if you have the header file. */ -#define HAVE_MEMORY_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MINIUPNPC_MINIUPNPC_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MINIUPNPC_UPNPCOMMANDS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MINIUPNPC_UPNPERRORS_H 1 - -/* Define this symbol if you have MSG_DONTWAIT */ -/* #undef HAVE_MSG_DONTWAIT */ - -/* Define this symbol if you have MSG_NOSIGNAL */ -/* #undef HAVE_MSG_NOSIGNAL */ - -/* Define if you have POSIX threads libraries and header files. */ -//#define HAVE_PTHREAD 1 - -/* Have PTHREAD_PRIO_INHERIT. */ -//#define HAVE_PTHREAD_PRIO_INHERIT 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDIO_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDLIB_H 1 - -/* Define to 1 if you have the `strerror_r' function. */ -/* #undef HAVE_STRERROR_R */ - -/* Define to 1 if you have the header file. */ -#define HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING_H 1 - -/* Define this symbol if the BSD sysctl(KERN_ARND) is available */ -/* #undef HAVE_SYSCTL_ARND */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_ENDIAN_H */ - -/* Define this symbol if the Linux getrandom system call is available */ -/* #undef HAVE_SYS_GETRANDOM */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_PRCTL_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_SELECT_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_TYPES_H 1 - -/* Define to 1 if you have the header file. */ -//#define HAVE_UNISTD_H 1 - -/* Define if the visibility attribute is supported. */ -#define HAVE_VISIBILITY_ATTRIBUTE 1 - -/* Define to the sub-directory where libtool stores uninstalled libraries. */ -#define LT_OBJDIR ".libs/" - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "https://github.com/bitcoin/bitcoin/issues" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "Bitcoin Core" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "Bitcoin Core 21.99.0" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "bitcoin" - -/* Define to the home page for this package. */ -#define PACKAGE_URL "https://bitcoincore.org/" - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "21.99.0" - -/* Define to necessary symbol if this constant uses a non-standard name on - your system. */ -/* #undef PTHREAD_CREATE_JOINABLE */ - -/* Define this symbol if the qt platform is cocoa */ -/* #undef QT_QPA_PLATFORM_COCOA */ - -/* Define this symbol if the minimal qt platform exists */ -#define QT_QPA_PLATFORM_MINIMAL 1 - -/* Define this symbol if the qt platform is windows */ -#define QT_QPA_PLATFORM_WINDOWS 1 - -/* Define this symbol if the qt platform is xcb */ -/* #undef QT_QPA_PLATFORM_XCB */ - -/* Define this symbol if qt plugins are static */ -#define QT_STATICPLUGIN 1 - -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* Define to 1 if strerror_r returns char *. */ -/* #undef STRERROR_R_CHAR_P */ - -/* Define this symbol to build in assembly routines */ -//#define USE_ASM 1 - -/* Define if dbus support should be compiled in */ -/* #undef USE_DBUS */ - -/* Define if QR support should be compiled in */ -//#define USE_QRCODE 1 - -/* UPnP support not compiled if undefined, otherwise value (0 or 1) determines - default state */ -//#define USE_UPNP 0 - -/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most - significant byte first (like Motorola and SPARC, unlike Intel). */ -#if defined AC_APPLE_UNIVERSAL_BUILD -# if defined __BIG_ENDIAN__ -# define WORDS_BIGENDIAN 1 -# endif -#else -# ifndef WORDS_BIGENDIAN -/* # undef WORDS_BIGENDIAN */ -# endif -#endif - -/* Enable large inode numbers on Mac OS X 10.5. */ -#ifndef _DARWIN_USE_64_BIT_INODE -# define _DARWIN_USE_64_BIT_INODE 1 -#endif - -/* Number of bits in a file offset, on hosts where this is settable. */ -#define _FILE_OFFSET_BITS 64 - -/* Define for large files, on AIX-style hosts. */ -/* #undef _LARGE_FILES */ - -/* Windows Universal Platform constraints */ -#if !defined(WINAPI_FAMILY) || (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP) -/* Either a desktop application without API restrictions, or and older system - before these macros were defined. */ - -/* ::wsystem is available */ -#define HAVE_SYSTEM 1 - -#endif // !WINAPI_FAMILY || WINAPI_FAMILY_DESKTOP_APP - -#endif //BITCOIN_BITCOIN_CONFIG_H diff --git a/build_msvc/bitcoin_config.h.in b/build_msvc/bitcoin_config.h.in new file mode 100644 index 000000000000..02d8fc41c2a6 --- /dev/null +++ b/build_msvc/bitcoin_config.h.in @@ -0,0 +1,155 @@ +// Copyright (c) 2018-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_BITCOIN_CONFIG_H +#define BITCOIN_BITCOIN_CONFIG_H + +/* Version Build */ +#define CLIENT_VERSION_BUILD $ + +/* Version is release */ +#define CLIENT_VERSION_IS_RELEASE $ + +/* Major version */ +#define CLIENT_VERSION_MAJOR $ + +/* Minor version */ +#define CLIENT_VERSION_MINOR $ + +/* Copyright holder(s) before %s replacement */ +#define COPYRIGHT_HOLDERS "The %s developers" + +/* Copyright holder(s) */ +#define COPYRIGHT_HOLDERS_FINAL "The Bitcoin Core developers" + +/* Replacement for %s in copyright holders string */ +#define COPYRIGHT_HOLDERS_SUBSTITUTION "Bitcoin Core" + +/* Copyright year */ +#define COPYRIGHT_YEAR $ + +/* Define to 1 to enable wallet functions */ +#define ENABLE_WALLET 1 + +/* Define to 1 to enable BDB wallet */ +#define USE_BDB 1 + +/* Define to 1 to enable SQLite wallet */ +#define USE_SQLITE 1 + +/* Define to 1 to enable ZMQ functions */ +#define ENABLE_ZMQ 1 + +/* define if external signer support is enabled (requires Boost::Process) */ +#define ENABLE_EXTERNAL_SIGNER /**/ + +/* Define this symbol if the consensus lib has been built */ +#define HAVE_CONSENSUS_LIB 1 + +/* Define to 1 if you have the declaration of `be16toh', and to 0 if you + don't. */ +#define HAVE_DECL_BE16TOH 0 + +/* Define to 1 if you have the declaration of `be32toh', and to 0 if you + don't. */ +#define HAVE_DECL_BE32TOH 0 + +/* Define to 1 if you have the declaration of `be64toh', and to 0 if you + don't. */ +#define HAVE_DECL_BE64TOH 0 + +/* Define to 1 if you have the declaration of `bswap_16', and to 0 if you + don't. */ +#define HAVE_DECL_BSWAP_16 0 + +/* Define to 1 if you have the declaration of `bswap_32', and to 0 if you + don't. */ +#define HAVE_DECL_BSWAP_32 0 + +/* Define to 1 if you have the declaration of `bswap_64', and to 0 if you + don't. */ +#define HAVE_DECL_BSWAP_64 0 + +/* Define to 1 if you have the declaration of `fork', and to 0 if you don't. + */ +#define HAVE_DECL_FORK 0 + +/* Define to 1 if you have the declaration of `htobe16', and to 0 if you + don't. */ +#define HAVE_DECL_HTOBE16 0 + +/* Define to 1 if you have the declaration of `htobe32', and to 0 if you + don't. */ +#define HAVE_DECL_HTOBE32 0 + +/* Define to 1 if you have the declaration of `htobe64', and to 0 if you + don't. */ +#define HAVE_DECL_HTOBE64 0 + +/* Define to 1 if you have the declaration of `htole16', and to 0 if you + don't. */ +#define HAVE_DECL_HTOLE16 0 + +/* Define to 1 if you have the declaration of `htole32', and to 0 if you + don't. */ +#define HAVE_DECL_HTOLE32 0 + +/* Define to 1 if you have the declaration of `htole64', and to 0 if you + don't. */ +#define HAVE_DECL_HTOLE64 0 + +/* Define to 1 if you have the declaration of `le16toh', and to 0 if you + don't. */ +#define HAVE_DECL_LE16TOH 0 + +/* Define to 1 if you have the declaration of `le32toh', and to 0 if you + don't. */ +#define HAVE_DECL_LE32TOH 0 + +/* Define to 1 if you have the declaration of `le64toh', and to 0 if you + don't. */ +#define HAVE_DECL_LE64TOH 0 + +/* Define to 1 if you have the declaration of `setsid', and to 0 if you don't. + */ +#define HAVE_DECL_SETSID 0 + +/* Define if the dllexport attribute is supported. */ +#define HAVE_DLLEXPORT_ATTRIBUTE 1 + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "https://github.com/bitcoin/bitcoin/issues" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "Bitcoin Core" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING $ + +/* Define to the home page for this package. */ +#define PACKAGE_URL "https://bitcoincore.org/" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION $ + +/* Define this symbol if the minimal qt platform exists */ +#define QT_QPA_PLATFORM_MINIMAL 1 + +/* Define this symbol if the qt platform is windows */ +#define QT_QPA_PLATFORM_WINDOWS 1 + +/* Define this symbol if qt plugins are static */ +#define QT_STATICPLUGIN 1 + +/* Windows Universal Platform constraints */ +#if !defined(WINAPI_FAMILY) || (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP) +/* Either a desktop application without API restrictions, or and older system + before these macros were defined. */ + +/* ::wsystem is available */ +#define HAVE_SYSTEM 1 + +#endif // !WINAPI_FAMILY || WINAPI_FAMILY_DESKTOP_APP + +#endif //BITCOIN_BITCOIN_CONFIG_H diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index c2c32af83808..b1204d0d5d15 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -24,7 +24,7 @@ {6190199c-6cf4-4dad-bfbd-93fa72a760c1} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} @@ -57,6 +57,8 @@ + + + diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj.in similarity index 81% rename from build_msvc/common.init.vcxproj rename to build_msvc/common.init.vcxproj.in index 6ea018d8463d..24d59221824d 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj.in @@ -14,7 +14,6 @@ true true $(Configuration) - x86-windows-static x64-windows-static @@ -35,20 +34,12 @@ Debug x64 - - Release - Win32 - - - Debug - Win32 - false false - v142 + @TOOLSET@ Unicode No $(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\ @@ -58,7 +49,7 @@ true true - v142 + @TOOLSET@ Unicode $(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\ $(Platform)\$(Configuration)\$(ProjectName)\ @@ -96,11 +87,11 @@ Level3 NotUsing - /utf-8 /Zc:__cplusplus /std:c++17 %(AdditionalOptions) - 4018;4244;4267;4334;4715;4805;4834 + /utf-8 /Zc:__cplusplus /std:c++20 %(AdditionalOptions) + 4018;4244;4267;4715;4805 true - _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions) - ..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories) + _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions) + ..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories) Console diff --git a/build_msvc/common.qt.init.vcxproj b/build_msvc/common.qt.init.vcxproj index ce66a7ab34b9..cc8063e545b8 100644 --- a/build_msvc/common.qt.init.vcxproj +++ b/build_msvc/common.qt.init.vcxproj @@ -2,7 +2,7 @@ - C:\Qt5.12.11_x64_static_vs2019_16101 + C:\Qt_static $(QtBaseDir)\plugins $(QtBaseDir)\lib $(QtBaseDir)\include diff --git a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in index b47d62b29587..482e4333f796 100644 --- a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in +++ b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in @@ -8,6 +8,7 @@ StaticLibrary + @SOURCE_FILES@ diff --git a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in b/build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in similarity index 100% rename from build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in rename to build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 96bb5843758b..9f9dc9d5fa04 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -24,6 +24,7 @@ + @@ -52,6 +53,7 @@ + @@ -78,6 +80,7 @@ + @@ -138,20 +141,6 @@ - - - _X86_;%(PreprocessorDefinitions) - $(QtIncludes);$(GeneratedFilesOutDir)\..;%(AdditionalIncludeDirectories) - - - - - - _X86_;%(PreprocessorDefinitions) - $(QtIncludes);$(GeneratedFilesOutDir)\..;%(AdditionalIncludeDirectories) - - - diff --git a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in index 6ec40461c2ad..adf4fa03540f 100644 --- a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in +++ b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in @@ -8,7 +8,6 @@ StaticLibrary - @SOURCE_FILES@ diff --git a/build_msvc/libleveldb/libleveldb.vcxproj b/build_msvc/libleveldb/libleveldb.vcxproj index 009be30decbf..2914eb2cfbb3 100644 --- a/build_msvc/libleveldb/libleveldb.vcxproj +++ b/build_msvc/libleveldb/libleveldb.vcxproj @@ -50,7 +50,7 @@ - HAVE_CRC32C=0;HAVE_SNAPPY=0;__STDC_LIMIT_MACROS;LEVELDB_IS_BIG_ENDIAN=0;_UNICODE;UNICODE;_CRT_NONSTDC_NO_DEPRECATE;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;%(PreprocessorDefinitions) + HAVE_CRC32C=0;HAVE_SNAPPY=0;LEVELDB_IS_BIG_ENDIAN=0;_UNICODE;UNICODE;_CRT_NONSTDC_NO_DEPRECATE;LEVELDB_PLATFORM_WINDOWS;%(PreprocessorDefinitions) 4244;4267 ..\..\src\leveldb;..\..\src\leveldb\include;%(AdditionalIncludeDirectories) diff --git a/build_msvc/libminisketch/libminisketch.vcxproj b/build_msvc/libminisketch/libminisketch.vcxproj new file mode 100644 index 000000000000..b34593fe5c71 --- /dev/null +++ b/build_msvc/libminisketch/libminisketch.vcxproj @@ -0,0 +1,38 @@ + + + + + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D} + + + StaticLibrary + + + + + + + + + + + + + + + + + + + + + + + 4060;4065;4146;4244;4267;4554 + HAVE_CLMUL;DISABLE_DEFAULT_FIELDS;ENABLE_FIELD_32;%(PreprocessorDefinitions) + + + + + + diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index f9b0a7975cb9..16ee32d87e10 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -8,6 +8,8 @@ StaticLibrary + + diff --git a/build_msvc/libsecp256k1_config.h b/build_msvc/libsecp256k1_config.h index 5978b9a0d96f..2b1a980e273f 100644 --- a/build_msvc/libsecp256k1_config.h +++ b/build_msvc/libsecp256k1_config.h @@ -8,25 +8,8 @@ #define BITCOIN_LIBSECP256K1_CONFIG_H #undef USE_ASM_X86_64 -#undef USE_ENDOMORPHISM -#undef USE_FIELD_10X26 -#undef USE_FIELD_5X52 -#undef USE_FIELD_INV_BUILTIN -#undef USE_FIELD_INV_NUM -#undef USE_NUM_GMP -#undef USE_NUM_NONE -#undef USE_SCALAR_4X64 -#undef USE_SCALAR_8X32 -#undef USE_SCALAR_INV_BUILTIN -#undef USE_SCALAR_INV_NUM - -#define USE_NUM_NONE 1 -#define USE_FIELD_INV_BUILTIN 1 -#define USE_SCALAR_INV_BUILTIN 1 -#define USE_FIELD_10X26 1 -#define USE_SCALAR_8X32 1 #define ECMULT_GEN_PREC_BITS 4 #define ECMULT_WINDOW_SIZE 15 -#endif /* BITCOIN_LIBSECP256K1_CONFIG_H */ +#endif // BITCOIN_LIBSECP256K1_CONFIG_H diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index d99b17d38127..ae48a52a2fe3 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,13 +9,13 @@ from shutil import copyfile SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src')) -DEFAULT_PLATFORM_TOOLSET = R'v141' +DEFAULT_PLATFORM_TOOLSET = R'v143' libs = [ 'libbitcoin_cli', 'libbitcoin_common', 'libbitcoin_crypto', - 'libbitcoin_server', + 'libbitcoin_node', 'libbitcoin_util', 'libbitcoin_wallet_tool', 'libbitcoin_wallet', @@ -50,20 +50,53 @@ def parse_makefile(makefile): lib_sources[current_lib] = [] break -def set_common_properties(toolset): - with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'r', encoding='utf-8') as rfile: - s = rfile.read() - s = re.sub('.*?', ''+toolset+'', s) - with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'w', encoding='utf-8',newline='\n') as wfile: - wfile.write(s) +def parse_config_into_btc_config(): + def find_between( s, first, last ): + try: + start = s.index( first ) + len( first ) + end = s.index( last, start ) + return s[start:end] + except ValueError: + return "" + + config_info = [] + with open(os.path.join(SOURCE_DIR,'../configure.ac'), encoding="utf8") as f: + for line in f: + if line.startswith("define"): + config_info.append(find_between(line, "(_", ")")) + + config_info = [c for c in config_info if not c.startswith("COPYRIGHT_HOLDERS")] + + config_dict = dict(item.split(", ") for item in config_info) + config_dict["PACKAGE_VERSION"] = f"\"{config_dict['CLIENT_VERSION_MAJOR']}.{config_dict['CLIENT_VERSION_MINOR']}.{config_dict['CLIENT_VERSION_BUILD']}\"" + version = config_dict["PACKAGE_VERSION"].strip('"') + config_dict["PACKAGE_STRING"] = f"\"Bitcoin Core {version}\"" + + with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h.in'), "r", encoding="utf8") as template_file: + template = template_file.readlines() + + for index, line in enumerate(template): + header = "" + if line.startswith("#define"): + header = line.split(" ")[1] + if header in config_dict: + template[index] = line.replace("$", f"{config_dict[header]}") + + with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), "w", encoding="utf8") as btc_config: + btc_config.writelines(template) + +def set_properties(vcxproj_filename, placeholder, content): + with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file: + with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: + vcxproj_file.write(vcxproj_in_file.read().replace(placeholder, content)) def main(): parser = argparse.ArgumentParser(description='Bitcoin-core msbuild configuration initialiser.') - parser.add_argument('-toolset', nargs='?',help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.' + parser.add_argument('-toolset', nargs='?', default=DEFAULT_PLATFORM_TOOLSET, + help='Optionally sets the msbuild platform toolset, e.g. v143 for Visual Studio 2022.' ' default is %s.'%DEFAULT_PLATFORM_TOOLSET) args = parser.parse_args() - if args.toolset: - set_common_properties(args.toolset) + set_properties(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), '@TOOLSET@', args.toolset) for makefile_name in os.listdir(SOURCE_DIR): if 'Makefile' in makefile_name: @@ -75,10 +108,8 @@ def main(): content += ' \n' content += ' $(IntDir)' + object_filename + '\n' content += ' \n' - with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file: - with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: - vcxproj_file.write(vcxproj_in_file.read().replace( - '@SOURCE_FILES@\n', content)) + set_properties(vcxproj_filename, '@SOURCE_FILES@\n', content) + parse_config_into_btc_config() copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h')) copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h')) diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 1d2c86b7ace6..3a2540d549ad 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -8,9 +8,11 @@ $(SolutionDir)$(Platform)\$(Configuration)\ + + @@ -19,6 +21,7 @@ + @@ -39,7 +42,7 @@ {2b4abff8-d1fd-4845-88c9-1f3c0a6512bf} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} @@ -51,6 +54,9 @@ {792d487f-f14c-49fc-a9de-3fc150f31c3f} + + {1e065f03-3566-47d0-8fa9-daa72b084e7d} + {18430fef-6b61-4c53-b396-718e02850f1b} @@ -87,6 +93,7 @@ + diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 5c4b777d5164..4182448ec356 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -16,8 +16,12 @@ + + + {542007e3-be0d-4b0d-a6b0-aa8813e2558d} + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} @@ -30,7 +34,7 @@ {6190199c-6cf4-4dad-bfbd-93fa72a760c1} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} diff --git a/build_msvc/testconsensus/testconsensus.cpp b/build_msvc/testconsensus/testconsensus.cpp deleted file mode 100644 index f3c8517130b3..000000000000 --- a/build_msvc/testconsensus/testconsensus.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2018-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include - -// bitcoin includes. -#include <..\src\script\bitcoinconsensus.h> -#include <..\src\primitives\transaction.h> -#include <..\src\script\script.h> -#include <..\src\streams.h> -#include <..\src\version.h> - -CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, int nValue = 0) -{ - CMutableTransaction txSpend; - txSpend.nVersion = 1; - txSpend.nLockTime = 0; - txSpend.vin.resize(1); - txSpend.vout.resize(1); - txSpend.vin[0].scriptWitness = scriptWitness; - txSpend.vin[0].prevout.hash = uint256(); - txSpend.vin[0].prevout.n = 0; - txSpend.vin[0].scriptSig = scriptSig; - txSpend.vin[0].nSequence = CTxIn::SEQUENCE_FINAL; - txSpend.vout[0].scriptPubKey = CScript(); - txSpend.vout[0].nValue = nValue; - - return txSpend; -} - -int main() -{ - std::cout << "bitcoinconsensus version: " << bitcoinconsensus_version() << std::endl; - - CScript pubKeyScript; - pubKeyScript << OP_1 << OP_0 << OP_1; - - int amount = 0; // 600000000; - - CScript scriptSig; - CScriptWitness scriptWitness; - CTransaction vanillaSpendTx = BuildSpendingTransaction(scriptSig, scriptWitness, amount); - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << vanillaSpendTx; - - bitcoinconsensus_error err; - auto op0Result = bitcoinconsensus_verify_script_with_amount(pubKeyScript.data(), pubKeyScript.size(), amount, stream.data(), stream.size(), 0, bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL, &err); - std::cout << "Op0 result: " << op0Result << ", error code " << err << std::endl; - - getchar(); - - return 0; -} diff --git a/build_msvc/testconsensus/testconsensus.vcxproj b/build_msvc/testconsensus/testconsensus.vcxproj deleted file mode 100644 index 776c40920abd..000000000000 --- a/build_msvc/testconsensus/testconsensus.vcxproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - - {E78473E9-B850-456C-9120-276301E04C06} - - - Application - $(SolutionDir)$(Platform)\$(Configuration)\ - - - - - - - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE} - - - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754} - - - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6} - - - - - - diff --git a/build_msvc/vcpkg.json b/build_msvc/vcpkg.json index 42b9a5d16fc0..86773d1fd3ce 100644 --- a/build_msvc/vcpkg.json +++ b/build_msvc/vcpkg.json @@ -3,13 +3,11 @@ "version-string": "1", "dependencies": [ "berkeleydb", - "boost-filesystem", "boost-multi-index", "boost-process", "boost-signals2", "boost-test", "sqlite3", - "double-conversion", { "name": "libevent", "features": ["thread"] diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 2c63a9efac8a..6f9a4709dde8 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -1,22 +1,27 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C ${CI_RETRY_EXE} apt-get update -${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq -update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 -update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 +${CI_RETRY_EXE} apt-get install -y python3-pip curl git gawk jq +( + # Temporary workaround for https://github.com/bitcoin/bitcoin/pull/26130#issuecomment-1260499544 + # Can be removed once the underlying image is bumped to something that includes git2.34 or later + sed -i -e 's/bionic/jammy/g' /etc/apt/sources.list + ${CI_RETRY_EXE} apt-get update + ${CI_RETRY_EXE} apt-get install -y --reinstall git +) -${CI_RETRY_EXE} pip3 install codespell==2.0.0 -${CI_RETRY_EXE} pip3 install flake8==3.8.3 -${CI_RETRY_EXE} pip3 install yq -${CI_RETRY_EXE} pip3 install mypy==0.781 +${CI_RETRY_EXE} pip3 install codespell==2.2.1 +${CI_RETRY_EXE} pip3 install flake8==4.0.1 +${CI_RETRY_EXE} pip3 install mypy==0.942 +${CI_RETRY_EXE} pip3 install pyzmq==22.3.0 ${CI_RETRY_EXE} pip3 install vulture==2.3 -SHELLCHECK_VERSION=v0.7.2 +SHELLCHECK_VERSION=v0.8.0 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index e38cfe8eefd2..826888b6cc1b 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,13 @@ export LC_ALL=C GIT_HEAD=$(git rev-parse HEAD) if [ -n "$CIRRUS_PR" ]; then - COMMIT_RANGE="$CIRRUS_BASE_SHA..$GIT_HEAD" - test/lint/commit-script-check.sh $COMMIT_RANGE + COMMIT_RANGE="${CIRRUS_BASE_SHA}..$GIT_HEAD" + echo + git log --no-merges --oneline "$COMMIT_RANGE" + echo + test/lint/commit-script-check.sh "$COMMIT_RANGE" +else + COMMIT_RANGE="SKIP_EMPTY_NOT_A_PR" fi export COMMIT_RANGE @@ -17,17 +22,22 @@ export COMMIT_RANGE # check with -r to not have to fetch all the remotes. test/lint/git-subtree-check.sh src/crypto/ctaes test/lint/git-subtree-check.sh src/secp256k1 -test/lint/git-subtree-check.sh src/univalue +test/lint/git-subtree-check.sh src/minisketch test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py -test/lint/lint-all.sh +test/lint/all-lint.py -if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then - git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit - ${CI_RETRY_EXE} gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $( ./contrib/verify-commits/trusted-sha512-root-commit + git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-git-root + mapfile -t KEYS < contrib/verify-commits/trusted-keys + git config user.email "ci@ci.ci" + git config user.name "ci" + ${CI_RETRY_EXE} gpg --keyserver hkps://keys.openpgp.org --recv-keys "${KEYS[@]}" && + ./contrib/verify-commits/verify-commits.py; fi - -echo -git log --no-merges --oneline $COMMIT_RANGE diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 8a9d808f5de9..5a150d5f8031 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -37,6 +37,7 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} +export RUN_TIDY=${RUN_TIDY:-false} export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # By how much to scale the test_runner timeouts (option --timeout-factor). # This is needed because some ci machines have slow CPU or disk, so sanitizers diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index f78a84eeac47..6732db36adf7 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export HOST=aarch64-linux-android -export PACKAGES="clang llvm unzip openjdk-8-jdk gradle" +export PACKAGES="unzip openjdk-8-jdk gradle" export CONTAINER_NAME=ci_android export DOCKER_NAME_TAG="ubuntu:focal" @@ -16,10 +16,10 @@ export RUN_FUNCTIONAL_TESTS=false export ANDROID_API_LEVEL=28 export ANDROID_BUILD_TOOLS_VERSION=28.0.3 -export ANDROID_NDK_VERSION=21.1.6352462 -export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-6609375_latest.zip +export ANDROID_NDK_VERSION=23.2.8568313 +export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-8512546_latest.zip export ANDROID_HOME="${DEPENDS_DIR}/SDKs/android" export ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}" export DEP_OPTS="ANDROID_SDK=${ANDROID_HOME} ANDROID_NDK=${ANDROID_NDK_HOME} ANDROID_API_LEVEL=${ANDROID_API_LEVEL} ANDROID_TOOLCHAIN_BIN=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/" -export BITCOIN_CONFIG="--disable-ccache" +export BITCOIN_CONFIG="--disable-tests --enable-gui-tests --disable-bench --disable-fuzz-binary --without-utils --without-libs --without-daemon" diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 8d2b70e549c9..932be4b43dc7 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,11 +18,11 @@ if [ -n "$QEMU_USER_CMD" ]; then fi export CONTAINER_NAME=ci_arm_linux # Use debian to avoid 404 apt errors when cross compiling -export DOCKER_NAME_TAG="debian:buster" +export DOCKER_NAME_TAG="debian:bullseye" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi" +export BITCOIN_CONFIG="--enable-reduce-exports CXXFLAGS=-Wno-psabi" diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 2ddb93290706..1ce3261f44e8 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -1,15 +1,16 @@ #!/usr/bin/env bash # -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu -export CONTAINER_NAME=ci_i686_centos_8 -export DOCKER_NAME_TAG=centos:8 -export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-zmq which patch lbzip2 dash rsync coreutils bison" +export CONTAINER_NAME=ci_i686_centos +export DOCKER_NAME_TAG=quay.io/centos/centos:stream8 +export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" +export PIP_PACKAGES="pyzmq" export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh new file mode 100755 index 000000000000..76de87d955dd --- /dev/null +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=i686-pc-linux-gnu +export CONTAINER_NAME=ci_i686_multiprocess +export DOCKER_NAME_TAG=ubuntu:20.04 +export PACKAGES="cmake python3 llvm clang g++-multilib" +export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" +export GOAL="install" +export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" +export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" +export TEST_RUNNER_EXTRA="--nosandbox" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 73ac09c1de1c..c4f22c8f9ea0 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well) -export HOST=x86_64-apple-darwin18 -export PACKAGES="cmake imagemagick librsvg2-bin libz-dev libtiff-tools libtinfo5 python3-setuptools xorriso" -export XCODE_VERSION=12.1 -export XCODE_BUILD_ID=12A7403 +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos +export HOST=x86_64-apple-darwin +export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso" +export XCODE_VERSION=12.2 +export XCODE_BUILD_ID=12B45b export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_native_arm64.sh similarity index 58% rename from ci/test/00_setup_env_mac_host.sh rename to ci/test/00_setup_env_mac_native_arm64.sh index c0d951a04171..cb0e13e77c78 100755 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_native_arm64.sh @@ -1,17 +1,16 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export HOST=x86_64-apple-darwin18 -export PIP_PACKAGES="zmq lief" +export HOST=arm64-apple-darwin +export PIP_PACKAGES="zmq" export GOAL="install" -export BITCOIN_CONFIG="--with-gui --enable-reduce-exports" +export BITCOIN_CONFIG="--with-gui --with-miniupnpc --with-natpmp --enable-reduce-exports" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" export CCACHE_SIZE=300M -export RUN_SECURITY_TESTS="true" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index ab185b6e7142..d8ebbbcdc9f3 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -1,14 +1,26 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 +# Only install BCC tracing packages in Cirrus CI. +if [[ "${CIRRUS_CI}" == "true" ]]; then + # We install an up-to-date 'bpfcc-tools' package from an untrusted PPA. + # This can be dropped with the next Ubuntu or Debian release that includes up-to-date packages. + # See the if-then in ci/test/04_install.sh too. + export ADD_UNTRUSTED_BPFCC_PPA=true + export BPFCC_PACKAGE="bpfcc-tools" +else + export ADD_UNTRUSTED_BPFCC_PPA=false + export BPFCC_PACKAGE="" +fi + export CONTAINER_NAME=ci_native_asan -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" -export DOCKER_NAME_TAG=ubuntu:hirsute +export PACKAGES="systemtap-sdt-dev clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export DOCKER_NAME_TAG=ubuntu:22.04 export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 58388fa92882..d7caec835906 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -1,18 +1,18 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export DOCKER_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang -ftrivial-auto-var-init=pattern' CXX='clang++ -ftrivial-auto-var-init=pattern'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index d5b28ca5cf02..21d6f3ea0a0b 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,10 +13,11 @@ LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export PACKAGES="clang-12 llvm-12 cmake" +# BDB generates false-positives and will be removed in future +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 2cf672b91ee8..97c530e19e64 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -1,19 +1,20 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export DOCKER_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev valgrind" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++" +# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 7bcf9f23a2b9..e1c03dcfedea 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,13 +11,13 @@ LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" -export BDB_PREFIX="${BASE_ROOT_DIR}/db4" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export PACKAGES="clang-12 llvm-12 cmake" +# BDB generates false-positives and will be removed in future +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh deleted file mode 100755 index 8869b2a08396..000000000000 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export CONTAINER_NAME=ci_native_multiprocess -export DOCKER_NAME_TAG=ubuntu:20.04 -export PACKAGES="cmake python3 python3-pip llvm clang" -export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" -export GOAL="install" -export BITCOIN_CONFIG="--enable-debug CC=clang CXX=clang++" # Use clang to avoid OOM -export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" -export PIP_PACKAGES="lief" diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh deleted file mode 100755 index d167c9198ad2..000000000000 --- a/ci/test/00_setup_env_native_nowallet.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export CONTAINER_NAME=ci_native_nowallet -export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md -export PACKAGES="python3-zmq clang-5.0 llvm-5.0" # Use clang-5 to test C++17 compatibility, see doc/dependencies.md -export DEP_OPTS="NO_WALLET=1" -export GOAL="install" -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-5.0 CXX=clang++-5.0" diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh new file mode 100755 index 000000000000..63560a5f5ccb --- /dev/null +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel +export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md +export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" # Use clang-8 to test C++17 compatibility, see doc/dependencies.md +export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" +export GOAL="install" +export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index b3e967c89866..8a6ea62d5c54 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -1,19 +1,19 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-7 can compile our c++17 and run our functional tests in python3, see doc/dependencies.md -export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" -export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" +export DOCKER_NAME_TAG=debian:buster # Check that buster gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md +export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" +export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports ---enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" +export DOWNLOAD_PREVIOUS_RELEASES="true" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ +--enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh new file mode 100755 index 000000000000..11a12e336a25 --- /dev/null +++ b/ci/test/00_setup_env_native_tidy.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export DOCKER_NAME_TAG="ubuntu:22.04" +export CONTAINER_NAME=ci_native_tidy +export PACKAGES="clang libclang-dev llvm-dev clang-tidy bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export NO_DEPENDS=1 +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export RUN_FUZZ_TESTS=false +export RUN_TIDY=true +export GOAL="install" +export BITCOIN_CONFIG="CC=clang CXX=clang++ --enable-c++20 --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" +export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index a5082bdaab37..6bf8391209fd 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export DOCKER_NAME_TAG=ubuntu:hirsute -export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" -export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" +export DOCKER_NAME_TAG=ubuntu:22.04 +export PACKAGES="clang-13 llvm-13 libc++abi-13-dev libc++-13-dev python3-zmq" +export DEP_OPTS="CC=clang-13 CXX='clang++-13 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread CC=clang-13 CXX='clang++-13 -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index e079a7057c87..d8c08fca39c5 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -1,15 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 +export DOCKER_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI +# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 51a0fd9117d8..136edb6662b7 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,13 +14,13 @@ export PACKAGES="python3-zmq" if [ -n "$QEMU_USER_CMD" ]; then # Likely cross-compiling, so install the needed gcc and qemu-user export DPKG_ADD_ARCH="s390x" - export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x libfontconfig1:s390x libxcb1:s390x" + export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x" fi # Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export DOCKER_NAME_TAG="debian:buster" -export RUN_UNIT_TESTS=true +export DOCKER_NAME_TAG="debian:bookworm" export TEST_RUNNER_ENV="LC_ALL=C" +export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb" +export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" # GUI tests disabled for now, see https://github.com/bitcoin/bitcoin/issues/23730 diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 4d5bde13fdda..360011355184 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,20 +1,16 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to win64 (Focal is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" -export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file" +export PACKAGES="python3 nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --disable-external-signer" - -# Compiler for MinGW-w64 causes false -Wreturn-type warning. -# See https://sourceforge.net/p/mingw-w64/bugs/306/ -export NO_WERROR=1 +export BITCOIN_CONFIG="--enable-reduce-exports --disable-external-signer --disable-gui-tests" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 2079d2ed2be7..4915797ca409 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,11 +10,6 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then export LC_ALL=C fi -if [ "$CI_OS_NAME" == "macos" ]; then - sudo -H pip3 install --upgrade pip - IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES -fi - # Create folders that are mounted into the docker mkdir -p "${CCACHE_DIR}" mkdir -p "${PREVIOUS_RELEASES_DIR}" @@ -32,6 +27,11 @@ export P_CI_DIR="$PWD" if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" + LOCAL_UID=$(id -u) + LOCAL_GID=$(id -g) + + # the name isn't important, so long as we use the same UID + LOCAL_USER=nonroot ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then @@ -39,6 +39,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then systemctl restart docker fi + # shellcheck disable=SC2086 DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ @@ -48,28 +49,57 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then --env-file /tmp/env \ --name $CONTAINER_NAME \ $DOCKER_NAME_TAG) - export DOCKER_CI_CMD_PREFIX="docker exec $DOCKER_ID" + + # Create a non-root user inside the container which matches the local user. + # + # This prevents the root user in the container modifying the local file system permissions + # on the mounted directories + docker exec "$DOCKER_ID" useradd -u "$LOCAL_UID" -o -m "$LOCAL_USER" + docker exec "$DOCKER_ID" groupmod -o -g "$LOCAL_GID" "$LOCAL_USER" + docker exec "$DOCKER_ID" chown -R "$LOCAL_USER":"$LOCAL_USER" "${BASE_ROOT_DIR}" + export DOCKER_CI_CMD_PREFIX_ROOT="docker exec -u 0 $DOCKER_ID" + export DOCKER_CI_CMD_PREFIX="docker exec -u $LOCAL_UID $DOCKER_ID" else echo "Running on host system without docker wrapper" fi -DOCKER_EXEC () { - $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" +CI_EXEC () { + $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd \"$P_CI_DIR\" && $*" +} +CI_EXEC_ROOT () { + $DOCKER_CI_CMD_PREFIX_ROOT bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd \"$P_CI_DIR\" && $*" } -export -f DOCKER_EXEC +export -f CI_EXEC +export -f CI_EXEC_ROOT if [ -n "$DPKG_ADD_ARCH" ]; then - DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" + CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH" fi -if [[ $DOCKER_NAME_TAG == centos* ]]; then - ${CI_RETRY_EXE} DOCKER_EXEC dnf -y install epel-release - ${CI_RETRY_EXE} DOCKER_EXEC dnf -y --allowerasing install $DOCKER_PACKAGES $PACKAGES +if [[ $DOCKER_NAME_TAG == *centos* ]]; then + ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y install epel-release + ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y --allowerasing install "$DOCKER_PACKAGES" "$PACKAGES" elif [ "$CI_USE_APT_INSTALL" != "no" ]; then - ${CI_RETRY_EXE} DOCKER_EXEC apt-get update - ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES - if [ -n "$PIP_PACKAGES" ]; then - ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then + # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages. + # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated + # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools + # package. + # TODO: drop this once we can use newer images in GCE + CI_EXEC_ROOT add-apt-repository ppa:hadret/bpfcc + fi + ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update + ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" +fi + +if [ -n "$PIP_PACKAGES" ]; then + if [ "$CI_OS_NAME" == "macos" ]; then + sudo -H pip3 install --upgrade pip + # shellcheck disable=SC2086 + IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + else + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES fi fi @@ -77,46 +107,60 @@ if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" else - DOCKER_EXEC free -m -h - DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) - DOCKER_EXEC echo $(lscpu | grep Endian) + CI_EXEC free -m -h + CI_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) + CI_EXEC echo "$(lscpu | grep Endian)" fi -DOCKER_EXEC echo "Free disk space:" -DOCKER_EXEC df -h - -if [ "$RUN_FUZZ_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - if [ ! -d ${DIR_QA_ASSETS} ]; then - DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} - fi +CI_EXEC echo "Free disk space:" +CI_EXEC df -h +if [ "$RUN_FUZZ_TESTS" = "true" ]; then export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + if [ ! -d "$DIR_FUZZ_IN" ]; then + CI_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}" + fi +elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ + if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then + CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA" + CI_EXEC curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" + fi fi -DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" +CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - DOCKER_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" - DOCKER_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" - DOCKER_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" - DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" - DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" - DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" + CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100" + CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100" + CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" + CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" +fi + +if [[ "${RUN_TIDY}" == "true" ]]; then + export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" + if [ ! -d "${DIR_IWYU}" ]; then + CI_EXEC "mkdir -p ${DIR_IWYU}/build/" + CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_14 ${DIR_IWYU}/include-what-you-use" + CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use" + CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS" + fi fi if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" - DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR + CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR" fi if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" - DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/ + CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/bins/" # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) # find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version) # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) # shellcheck disable=SC1010 - DOCKER_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) $BASE_SCRATCH_DIR/bins/\$util\; done + CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) "${BASE_SCRATCH_DIR}/bins/\$util"\; done # Print BusyBox version - DOCKER_EXEC patch --help + CI_EXEC patch --help fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 8dd489d7f8fe..dd2b43d38bf9 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,53 +8,46 @@ export LC_ALL=C.UTF-8 # Make sure default datadir does not exist and is never read by creating a dummy file if [ "$CI_OS_NAME" == "macos" ]; then - echo > $HOME/Library/Application\ Support/Bitcoin + echo > "${HOME}/Library/Application Support/Bitcoin" else - DOCKER_EXEC echo \> \$HOME/.bitcoin + CI_EXEC echo \> \$HOME/.bitcoin + CI_EXEC_ROOT echo \> \$HOME/.bitcoin fi -DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources +CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" -OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" -OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" -if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then - DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" -fi - -if [ -n "$ANDROID_TOOLS_URL" ]; then - ANDROID_TOOLS_PATH=$DEPENDS_DIR/sdk-sources/android-tools.zip - - DOCKER_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" - DOCKER_EXEC mkdir -p "${ANDROID_HOME}/cmdline-tools" - DOCKER_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "${ANDROID_HOME}/cmdline-tools" - DOCKER_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" +if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" + if [ ! -f "$OSX_SDK_PATH" ]; then + CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" + fi + CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB - # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for - # details. - DOCKER_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then + ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip + if [ ! -f "$ANDROID_TOOLS_PATH" ]; then + CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" + fi + CI_EXEC mkdir -p "$ANDROID_HOME" + CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME" + CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" fi -if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then - DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" -fi -if [[ $HOST = *-mingw32 ]]; then - DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) -fi if [ -z "$NO_DEPENDS" ]; then - if [[ $DOCKER_NAME_TAG == centos* ]]; then + if [[ $DOCKER_NAME_TAG == *centos* ]]; then # CentOS has problems building the depends if the config shell is not explicitly set # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to # an error as the first command is executed) - SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/bash" + SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/dash" else SHELL_OPTS="CONFIG_SHELL=" fi - DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS + CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1 fi -if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then - DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" +if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then + CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index a42cd6cee143..573d50678a72 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -1,55 +1,68 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -if [ -n "$ANDROID_TOOLS_URL" ]; then - DOCKER_EXEC make distclean || true - DOCKER_EXEC ./autogen.sh - DOCKER_EXEC ./configure $BITCOIN_CONFIG --prefix=$DEPENDS_DIR/aarch64-linux-android || ( (DOCKER_EXEC cat config.log) && false) - DOCKER_EXEC "cd src/qt && make $MAKEJOBS && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" - exit 0 +BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" +if [ -z "$NO_DEPENDS" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" fi - -BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" if [ -z "$NO_WERROR" ]; then BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" fi -DOCKER_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" + +CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" +PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" + +if [ -n "$ANDROID_TOOLS_URL" ]; then + CI_EXEC make distclean || true + CI_EXEC ./autogen.sh + CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) + CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" + CI_EXEC "${PRINT_CCACHE_STATISTICS}" + exit 0 +fi + +BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" if [ -n "$CONFIG_SHELL" ]; then - DOCKER_EXEC "$CONFIG_SHELL" -c "./autogen.sh" + CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" else - DOCKER_EXEC ./autogen.sh + CI_EXEC ./autogen.sh fi -DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}" +CI_EXEC mkdir -p "${BASE_BUILD_DIR}" export P_CI_DIR="${BASE_BUILD_DIR}" -DOCKER_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) +CI_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) -DOCKER_EXEC make distdir VERSION=$HOST +CI_EXEC make distdir VERSION="$HOST" export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST" -DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) +CI_EXEC ./configure --cache-file=../config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) set -o errtrace -trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR +trap 'CI_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then # MemorySanitizer (MSAN) does not support tracking memory initialization done by # using the Linux getrandom syscall. Avoid using getrandom by undefining # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for # details. - DOCKER_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' + CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' +fi + +if [[ "${RUN_TIDY}" == "true" ]]; then + MAYBE_BEAR="bear --config src/.bear-tidy-config" + MAYBE_TOKEN="--" fi -DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) +CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) -DOCKER_EXEC "ccache --version | head -n 1 && ccache --show-stats" -DOCKER_EXEC du -sh "${DEPENDS_DIR}"/*/ -DOCKER_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" +CI_EXEC "${PRINT_CCACHE_STATISTICS}" +CI_EXEC du -sh "${DEPENDS_DIR}"/*/ +CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 194b14beabac..46312b50eb21 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,38 +8,77 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped - DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 - DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 - DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" + CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped - DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 - DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 - DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" + CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi if [ -n "$USE_VALGRIND" ]; then - DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" + CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" fi if [ "$RUN_UNIT_TESTS" = "true" ]; then - DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 + CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" make "$MAKEJOBS" check VERBOSE=1 fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite + CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}/bin/test_bitcoin" --catch_system_errors=no -l test_suite fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast + CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast +fi + +if [ "${RUN_TIDY}" = "true" ]; then + set -eo pipefail + export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/" + ( CI_EXEC run-clang-tidy -quiet "${MAKEJOBS}" ) | grep -C5 "error" + export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/" + CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\ + " src/common/url.cpp"\ + " src/compat"\ + " src/dbwrapper.cpp"\ + " src/init"\ + " src/kernel"\ + " src/node/chainstate.cpp"\ + " src/node/chainstatemanager_args.cpp"\ + " src/node/mempool_args.cpp"\ + " src/node/validation_cache_args.cpp"\ + " src/policy/feerate.cpp"\ + " src/policy/packages.cpp"\ + " src/policy/settings.cpp"\ + " src/primitives/transaction.cpp"\ + " src/rpc/fees.cpp"\ + " src/rpc/signmessage.cpp"\ + " src/test/fuzz/txorphan.cpp"\ + " src/test/fuzz/util/"\ + " src/util/bip32.cpp"\ + " src/util/bytevectorhash.cpp"\ + " src/util/check.cpp"\ + " src/util/error.cpp"\ + " src/util/getuniquepath.cpp"\ + " src/util/hasher.cpp"\ + " src/util/message.cpp"\ + " src/util/moneystr.cpp"\ + " src/util/serfloat.cpp"\ + " src/util/spanparsing.cpp"\ + " src/util/strencodings.cpp"\ + " src/util/string.cpp"\ + " src/util/syserror.cpp"\ + " src/util/threadinterrupt.cpp"\ + " src/zmq"\ + " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" fi if [ "$RUN_SECURITY_TESTS" = "true" ]; then - DOCKER_EXEC make test-security-check + CI_EXEC make test-security-check fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN} + CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}" fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index 2cd7c8cec290..eb31edbce8c4 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}; do # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do echo "Wrap $b ..." mv "$b" "${b}_orig" echo '#!/usr/bin/env bash' > "$b" diff --git a/ci/test/wrap-valgrind.sh b/ci/test/wrap-valgrind.sh index 6b3e6eb7e741..27754831848b 100755 --- a/ci/test/wrap-valgrind.sh +++ b/ci/test/wrap-valgrind.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 for b_name in "${BASE_OUTDIR}/bin"/*; do # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do echo "Wrap $b ..." mv "$b" "${b}_orig" echo '#!/usr/bin/env bash' > "$b" diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 82964897e133..1662f8f6a335 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}.exe; do # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do if (file "$b" | grep "Windows"); then echo "Wrap $b ..." mv "$b" "${b}_orig" diff --git a/ci/test/wrapped-cl.bat b/ci/test/wrapped-cl.bat new file mode 100644 index 000000000000..fc2a604c580b --- /dev/null +++ b/ci/test/wrapped-cl.bat @@ -0,0 +1 @@ +ccache cl %* diff --git a/configure.ac b/configure.ac index fa1b580cf582..541f31e3f0fc 100644 --- a/configure.ac +++ b/configure.ac @@ -1,10 +1,10 @@ AC_PREREQ([2.69]) -define(_CLIENT_VERSION_MAJOR, 21) +define(_CLIENT_VERSION_MAJOR, 24) define(_CLIENT_VERSION_MINOR, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2021) +define(_COPYRIGHT_YEAR, 2022) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin Core]]) AC_INIT([Bitcoin Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_BUILD)m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/bitcoin/bitcoin/issues],[bitcoin],[https://bitcoincore.org/]) @@ -13,25 +13,38 @@ AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) -m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) +m4_ifndef([PKG_PROG_PKG_CONFIG], [m4_fatal([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) PKG_PROG_PKG_CONFIG -if test "x$PKG_CONFIG" = x; then +if test "$PKG_CONFIG" = ""; then AC_MSG_ERROR([pkg-config not found]) fi +# When compiling with depends, the `PKG_CONFIG_PATH` and `PKG_CONFIG_LIBDIR` variables, +# being set in a `config.site` file, are not exported to let the `--config-cache` option +# work properly. +if test -n "$PKG_CONFIG_PATH"; then + PKG_CONFIG="env PKG_CONFIG_PATH=$PKG_CONFIG_PATH $PKG_CONFIG" +fi +if test -n "$PKG_CONFIG_LIBDIR"; then + PKG_CONFIG="env PKG_CONFIG_LIBDIR=$PKG_CONFIG_LIBDIR $PKG_CONFIG" +fi + BITCOIN_DAEMON_NAME=bitcoind BITCOIN_GUI_NAME=bitcoin-qt +BITCOIN_TEST_NAME=test_bitcoin BITCOIN_CLI_NAME=bitcoin-cli BITCOIN_TX_NAME=bitcoin-tx BITCOIN_UTIL_NAME=bitcoin-util +BITCOIN_CHAINSTATE_NAME=bitcoin-chainstate BITCOIN_WALLET_TOOL_NAME=bitcoin-wallet dnl Multi Process BITCOIN_MP_NODE_NAME=bitcoin-node BITCOIN_MP_GUI_NAME=bitcoin-gui dnl Unless the user specified ARFLAGS, force it to be cr -AC_ARG_VAR(ARFLAGS, [Flags for the archiver, defaults to if not set]) -if test "x${ARFLAGS+set}" != "xset"; then +dnl This is also the default as-of libtool 2.4.7 +AC_ARG_VAR([ARFLAGS], [Flags for the archiver, defaults to if not set]) +if test "${ARFLAGS+set}" != "set"; then ARFLAGS="cr" fi @@ -41,21 +54,16 @@ AH_TOP([#ifndef BITCOIN_CONFIG_H]) AH_TOP([#define BITCOIN_CONFIG_H]) AH_BOTTOM([#endif //BITCOIN_CONFIG_H]) -dnl faketime breaks configure and is only needed for make. Disable it here. -unset FAKETIME - dnl Automake init set-up and checks AM_INIT_AUTOMAKE([1.13 no-define subdir-objects foreign]) -dnl faketime messes with timestamps and causes configure to be re-run. -dnl --disable-maintainer-mode can be used to bypass this. AM_MAINTAINER_MODE([enable]) dnl make the compilation flags quiet unless V=1 is used AM_SILENT_RULES([yes]) dnl Compiler checks (here before libtool). -if test "x${CXXFLAGS+set}" = "xset"; then +if test "${CXXFLAGS+set}" = "set"; then CXXFLAGS_overridden=yes else CXXFLAGS_overridden=no @@ -71,48 +79,59 @@ case $host in ;; esac +AC_ARG_WITH([seccomp], + [AS_HELP_STRING([--with-seccomp], + [enable experimental syscall sandbox feature (-sandbox), default is yes if seccomp-bpf is detected under Linux x86_64])], + [seccomp_found=$withval], + [seccomp_found=auto]) + +AC_ARG_ENABLE([c++20], + [AS_HELP_STRING([--enable-c++20], + [enable compilation in c++20 mode (disabled by default)])], + [use_cxx20=$enableval], + [use_cxx20=no]) + dnl Require C++17 compiler (no GNU extensions) +if test "$use_cxx20" = "no"; then AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) +else +AX_CXX_COMPILE_STDCXX([20], [noext], [mandatory]) +fi -dnl Check if -latomic is required for -CHECK_ATOMIC +dnl check if additional link flags are required for std::filesystem +CHECK_FILESYSTEM dnl Unless the user specified OBJCXX, force it to be the same as CXX. This ensures dnl that we get the same -std flags for both. m4_ifdef([AC_PROG_OBJCXX],[ -if test "x${OBJCXX+set}" = "x"; then +if test "${OBJCXX+set}" = ""; then OBJCXX="${CXX}" fi AC_PROG_OBJCXX ]) -dnl Since libtool 1.5.2 (released 2004-01-25), on Linux libtool no longer -dnl sets RPATH for any directories in the dynamic linker search path. -dnl See more: https://wiki.debian.org/RpathIssue -LT_PREREQ([1.5.2]) +dnl OpenBSD ships with 2.4.2 +LT_PREREQ([2.4.2]) dnl Libtool init checks. -LT_INIT([pic-only]) +LT_INIT([pic-only win32-dll]) dnl Check/return PATH for base programs. -AC_PATH_TOOL(AR, ar) -AC_PATH_TOOL(RANLIB, ranlib) -AC_PATH_TOOL(STRIP, strip) -AC_PATH_TOOL(GCOV, gcov) -AC_PATH_TOOL(LLVM_COV, llvm-cov) -AC_PATH_PROG(LCOV, lcov) +AC_PATH_TOOL([AR], [ar]) +AC_PATH_TOOL([GCOV], [gcov]) +AC_PATH_TOOL([LLVM_COV], [llvm-cov]) +AC_PATH_PROG([LCOV], [lcov]) dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3 python]) -AC_PATH_PROG(GENHTML, genhtml) +AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3 python]) +AC_PATH_PROG([GENHTML], [genhtml]) AC_PATH_PROG([GIT], [git]) -AC_PATH_PROG(CCACHE,ccache) -AC_PATH_PROG(XGETTEXT,xgettext) -AC_PATH_PROG(HEXDUMP,hexdump) -AC_PATH_TOOL(CPPFILT, c++filt) -AC_PATH_TOOL(OBJCOPY, objcopy) -AC_PATH_PROG(DOXYGEN, doxygen) +AC_PATH_PROG([CCACHE], [ccache]) +AC_PATH_PROG([XGETTEXT], [xgettext]) +AC_PATH_PROG([HEXDUMP], [hexdump]) +AC_PATH_TOOL([OBJCOPY], [objcopy]) +AC_PATH_PROG([DOXYGEN], [doxygen]) AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"]) -AC_ARG_VAR(PYTHONPATH, Augments the default search path for python module files) +AC_ARG_VAR([PYTHONPATH], [Augments the default search path for python module files]) AC_ARG_ENABLE([wallet], [AS_HELP_STRING([--disable-wallet], @@ -132,11 +151,11 @@ AC_ARG_WITH([bdb], [use_bdb=$withval], [use_bdb=auto]) -AC_ARG_ENABLE([ebpf], - [AS_HELP_STRING([--enable-ebpf], - [enable eBPF tracing (default is yes if sys/sdt.h is found)])], - [use_ebpf=$enableval], - [use_ebpf=yes]) +AC_ARG_ENABLE([usdt], + [AS_HELP_STRING([--enable-usdt], + [enable tracepoints for Userspace, Statically Defined Tracing (default is yes if sys/sdt.h is found)])], + [use_usdt=$enableval], + [use_usdt=yes]) AC_ARG_WITH([miniupnpc], [AS_HELP_STRING([--with-miniupnpc], @@ -240,15 +259,9 @@ AC_ARG_ENABLE([lcov-branch-coverage], [use_lcov_branch=yes], [use_lcov_branch=no]) -AC_ARG_ENABLE([glibc-back-compat], - [AS_HELP_STRING([--enable-glibc-back-compat], - [enable backwards compatibility with glibc])], - [use_glibc_compat=$enableval], - [use_glibc_compat=no]) - AC_ARG_ENABLE([threadlocal], [AS_HELP_STRING([--enable-threadlocal], - [enable features that depend on the c++ thread_local keyword (currently just thread names in debug logs). (default is to enabled if there is platform support and glibc-back-compat is not enabled)])], + [enable features that depend on the c++ thread_local keyword (currently just thread names in debug logs). (default is to enable if there is platform support)])], [use_thread_local=$enableval], [use_thread_local=auto]) @@ -258,16 +271,10 @@ AC_ARG_ENABLE([asm], [use_asm=$enableval], [use_asm=yes]) -if test "x$use_asm" = xyes; then - AC_DEFINE(USE_ASM, 1, [Define this symbol to build in assembly routines]) +if test "$use_asm" = "yes"; then + AC_DEFINE([USE_ASM], [1], [Define this symbol to build in assembly routines]) fi -AC_ARG_WITH([system-univalue], - [AS_HELP_STRING([--with-system-univalue], - [Build with system UniValue (default is no)])], - [system_univalue=$withval], - [system_univalue=no] -) AC_ARG_ENABLE([zmq], [AS_HELP_STRING([--disable-zmq], [disable ZMQ notifications])], @@ -296,7 +303,7 @@ AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [do not install man pages (default is to install)])],, enable_man=yes) -AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) +AM_CONDITIONAL([ENABLE_MAN], [test "$enable_man" != "no"]) dnl Enable debug AC_ARG_ENABLE([debug], @@ -321,14 +328,19 @@ AC_ARG_ENABLE([gprof], dnl Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], - [Treat certain compiler warnings as errors (default is no)])], + [Treat compiler warnings as errors (default is no)])], [enable_werror=$enableval], [enable_werror=no]) AC_ARG_ENABLE([external-signer], - [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is yes, requires Boost::Process)])], + [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is auto, requires Boost::Process)])], [use_external_signer=$enableval], - [use_external_signer=yes]) + [use_external_signer=auto]) + +AC_ARG_ENABLE([lto], + [AS_HELP_STRING([--enable-lto],[build using LTO (default is no)])], + [enable_lto=$enableval], + [enable_lto=no]) AC_LANG_PUSH([C++]) @@ -339,7 +351,7 @@ dnl Note that this is not necessarily a check to see if -Werror is supported, bu dnl a compile with -Werror can succeed. This is important because the compiler may already be dnl warning about something unrelated, for example about some path issue. If that is the case, dnl -Werror cannot be used because all of those warnings would be turned into errors. -AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) +AX_CHECK_COMPILE_FLAG([-Werror], [CXXFLAG_WERROR="-Werror"], [CXXFLAG_WERROR=""]) dnl Check for a flag to turn linker warnings into errors. When flags are passed to linkers via the dnl compiler driver using a -Wl,-foo flag, linker warnings may be swallowed rather than bubbling up. @@ -348,42 +360,51 @@ dnl dnl LDFLAG_WERROR Should only be used when testing -Wl,* case $host in *darwin*) - AX_CHECK_LINK_FLAG([-Wl,-fatal_warnings],[LDFLAG_WERROR="-Wl,-fatal_warnings"],[LDFLAG_WERROR=""]) + AX_CHECK_LINK_FLAG([-Wl,-fatal_warnings], [LDFLAG_WERROR="-Wl,-fatal_warnings"], [LDFLAG_WERROR=""]) ;; *) - AX_CHECK_LINK_FLAG([-Wl,--fatal-warnings],[LDFLAG_WERROR="-Wl,--fatal-warnings"],[LDFLAG_WERROR=""]) + AX_CHECK_LINK_FLAG([-Wl,--fatal-warnings], [LDFLAG_WERROR="-Wl,--fatal-warnings"], [LDFLAG_WERROR=""]) ;; esac -if test "x$enable_debug" = xyes; then - dnl Clear default -g -O2 flags - if test "x$CXXFLAGS_overridden" = xno; then - CXXFLAGS="" +if test "$enable_debug" = "yes"; then + dnl If debugging is enabled, and the user hasn't overridden CXXFLAGS, clear + dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up + dnl with "-O0 -g3 -g -O2". + if test "$CXXFLAGS_overridden" = "no"; then + CXXFLAGS="" fi dnl Disable all optimizations - AX_CHECK_COMPILE_FLAG([-O0], [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"]],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-O0], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"], [], [$CXXFLAG_WERROR]) dnl Prefer -g3, fall back to -g if that is unavailable. AX_CHECK_COMPILE_FLAG( [-g3], - [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g3"]], - [AX_CHECK_COMPILE_FLAG([-g],[[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g"]],,[[$CXXFLAG_WERROR]])], - [[$CXXFLAG_WERROR]]) - - AX_CHECK_PREPROC_FLAG([-DDEBUG],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"]],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"]],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-ftrapv],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"],,[[$CXXFLAG_WERROR]]) + [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g3"], + [AX_CHECK_COMPILE_FLAG([-g], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g"], [], [$CXXFLAG_WERROR])], + [$CXXFLAG_WERROR]) + + AX_CHECK_PREPROC_FLAG([-DDEBUG], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKCONTENTION], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKCONTENTION"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DRPC_DOC_CHECK], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DRPC_DOC_CHECK"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-ftrapv], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"], [], [$CXXFLAG_WERROR]) fi -if test x$use_sanitizers != x; then +if test "$enable_lto" = "yes"; then + AX_CHECK_COMPILE_FLAG([-flto], [LTO_CXXFLAGS="$LTO_CXXFLAGS -flto"], [AC_MSG_ERROR([compile failed with -flto])], [$CXXFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-flto], [LTO_LDFLAGS="$LTO_LDFLAGS -flto"], [AC_MSG_ERROR([link failed with -flto])], [$CXXFLAG_WERROR]) +fi + +if test "$use_sanitizers" != ""; then dnl First check if the compiler accepts flags. If an incompatible pair like dnl -fsanitize=address,thread is used here, this check will fail. This will also dnl fail if a bad argument is passed, e.g. -fsanitize=undfeined AX_CHECK_COMPILE_FLAG( - [[-fsanitize=$use_sanitizers]], - [[SANITIZER_CXXFLAGS=-fsanitize=$use_sanitizers]], + [-fsanitize=$use_sanitizers], + [SANITIZER_CXXFLAGS="-fsanitize=$use_sanitizers"], [AC_MSG_ERROR([compiler did not accept requested flags])]) dnl Some compilers (e.g. GCC) require additional libraries like libasan, @@ -392,8 +413,8 @@ if test x$use_sanitizers != x; then dnl the sanitize flags are supported by the compiler but the actual sanitizer dnl libs are missing. AX_CHECK_LINK_FLAG( - [[-fsanitize=$use_sanitizers]], - [[SANITIZER_LDFLAGS=-fsanitize=$use_sanitizers]], + [-fsanitize=$use_sanitizers], + [SANITIZER_LDFLAGS="-fsanitize=$use_sanitizers"], [AC_MSG_ERROR([linker did not accept requested flags, you are missing required libraries])], [], [AC_LANG_PROGRAM([[ @@ -405,98 +426,101 @@ if test x$use_sanitizers != x; then fi ERROR_CXXFLAGS= -if test "x$enable_werror" = "xyes"; then - if test "x$CXXFLAG_WERROR" = "x"; then - AC_MSG_ERROR("enable-werror set but -Werror is not usable") +if test "$enable_werror" = "yes"; then + if test "$CXXFLAG_WERROR" = ""; then + AC_MSG_ERROR([enable-werror set but -Werror is not usable]) fi - AX_CHECK_COMPILE_FLAG([-Werror=gnu],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=gnu"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=shadow-field],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=shadow-field"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=thread-safety],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=range-loop-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=range-loop-analysis"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=return-type],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=return-type"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=conditional-uninitialized],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=conditional-uninitialized"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=sign-compare],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=sign-compare"],,[[$CXXFLAG_WERROR]]) - dnl -Wsuggest-override is broken with GCC before 9.2 - dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 - AX_CHECK_COMPILE_FLAG([-Werror=suggest-override],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=suggest-override"],,[[$CXXFLAG_WERROR]], - [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) - AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=mismatched-tags], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=mismatched-tags"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Werror=implicit-fallthrough], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=implicit-fallthrough"], [], [$CXXFLAG_WERROR]) + ERROR_CXXFLAGS=$CXXFLAG_WERROR - if test x$suppress_external_warnings != xno ; then - AX_CHECK_COMPILE_FLAG([-Werror=documentation],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=documentation"],,[[$CXXFLAG_WERROR]]) - fi + dnl -Wreturn-type is broken in GCC for MinGW-w64. + dnl https://sourceforge.net/p/mingw-w64/bugs/306/ + AX_CHECK_COMPILE_FLAG([-Werror=return-type], [], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Wno-error=return-type"], [$CXXFLAG_WERROR], + [AC_LANG_SOURCE([[#include + int f(){ assert(false); }]])]) fi -if test "x$CXXFLAGS_overridden" = "xno"; then - AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wextra],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wgnu],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"],,[[$CXXFLAG_WERROR]]) +if test "$CXXFLAGS_overridden" = "no"; then + AX_CHECK_COMPILE_FLAG([-Wall], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wextra], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wgnu], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"], [], [$CXXFLAG_WERROR]) dnl some compilers will ignore -Wformat-security without -Wformat, so just combine the two here. - AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wshadow-field],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wswitch],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wswitch"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wthread-safety],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wunused-member-function],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsign-compare"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wduplicated-branches],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wduplicated-cond],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wlogical-op],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], + AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wvla], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wshadow-field], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wthread-safety], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wredundant-decls], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wunused-member-function], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wdate-time], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-branches], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-cond], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wlogical-op], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"], [], [$CXXFLAG_WERROR]) + dnl -Wsuggest-override is broken with GCC before 9.2 + dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 + AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"], [], [$CXXFLAG_WERROR], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) - AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) - if test x$suppress_external_warnings != xno ; then - AX_CHECK_COMPILE_FLAG([-Wdocumentation],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"],,[[$CXXFLAG_WERROR]]) + if test "$suppress_external_warnings" != "no" ; then + AX_CHECK_COMPILE_FLAG([-Wdocumentation], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"], [], [$CXXFLAG_WERROR]) fi dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and dnl set the -Wno-foo case if it works. - AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wself-assign],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) - if test x$suppress_external_warnings != xyes ; then - AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunused-parameter], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wself-assign], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"], [], [$CXXFLAG_WERROR]) + if test "$suppress_external_warnings" != "yes" ; then + AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"], [], [$CXXFLAG_WERROR]) fi fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. -AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers],[[CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"]],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) +enable_arm_crc=no +enable_arm_shani=no enable_sse42=no enable_sse41=no enable_avx2=no -enable_shani=no +enable_x86_shani=no -if test "x$use_asm" = "xyes"; then +if test "$use_asm" = "yes"; then dnl Check for optional instruction set support. Enabling these does _not_ imply that all code will dnl be compiled with them, rather that specific objects/libs may use them after checking for runtime dnl compatibility. dnl x86 -AX_CHECK_COMPILE_FLAG([-msse4.2],[[SSE42_CXXFLAGS="-msse4.2"]],,[[$CXXFLAG_WERROR]]) -AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERROR]]) -AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]]) -AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-msse4.2], [SSE42_CXXFLAGS="-msse4.2"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-msse4.1], [SSE41_CXXFLAGS="-msse4.1"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-mavx -mavx2], [AVX2_CXXFLAGS="-mavx -mavx2"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-msse4 -msha], [X86_SHANI_CXXFLAGS="-msse4 -msha"], [], [$CXXFLAG_WERROR]) + +enable_clmul= +AX_CHECK_COMPILE_FLAG([-mpclmul], [enable_clmul=yes], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([ + #include + #include +], [ + __m128i a = _mm_cvtsi64_si128((uint64_t)7); + __m128i b = _mm_clmulepi64_si128(a, a, 37); + __m128i c = _mm_srli_epi64(b, 41); + __m128i d = _mm_xor_si128(b, c); + uint64_t e = _mm_cvtsi128_si64(d); + return e == 0; +])]) + +if test "$enable_clmul" = "yes"; then + CLMUL_CXXFLAGS="-mpclmul" + AC_DEFINE([HAVE_CLMUL], [1], [Define this symbol if clmul instructions can be used]) +fi TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS" -AC_MSG_CHECKING(for SSE4.2 intrinsics) +CXXFLAGS="$SSE42_CXXFLAGS $CXXFLAGS" +AC_MSG_CHECKING([for SSE4.2 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #if defined(_MSC_VER) @@ -511,14 +535,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ l = _mm_crc32_u64(l, 0); return l; ]])], - [ AC_MSG_RESULT(yes); enable_sse42=yes], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_sse42=yes], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $SSE41_CXXFLAGS" -AC_MSG_CHECKING(for SSE4.1 intrinsics) +CXXFLAGS="$SSE41_CXXFLAGS $CXXFLAGS" +AC_MSG_CHECKING([for SSE4.1 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include @@ -526,14 +550,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m128i l = _mm_set1_epi32(0); return _mm_extract_epi32(l, 3); ]])], - [ AC_MSG_RESULT(yes); enable_sse41=yes; AC_DEFINE(ENABLE_SSE41, 1, [Define this symbol to build code that uses SSE4.1 intrinsics]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_sse41=yes; AC_DEFINE([ENABLE_SSE41], [1], [Define this symbol to build code that uses SSE4.1 intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $AVX2_CXXFLAGS" -AC_MSG_CHECKING(for AVX2 intrinsics) +CXXFLAGS="$AVX2_CXXFLAGS $CXXFLAGS" +AC_MSG_CHECKING([for AVX2 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include @@ -541,14 +565,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m256i l = _mm256_set1_epi32(0); return _mm256_extract_epi32(l, 7); ]])], - [ AC_MSG_RESULT(yes); enable_avx2=yes; AC_DEFINE(ENABLE_AVX2, 1, [Define this symbol to build code that uses AVX2 intrinsics]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_avx2=yes; AC_DEFINE([ENABLE_AVX2], [1], [Define this symbol to build code that uses AVX2 intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $SHANI_CXXFLAGS" -AC_MSG_CHECKING(for SHA-NI intrinsics) +CXXFLAGS="$X86_SHANI_CXXFLAGS $CXXFLAGS" +AC_MSG_CHECKING([for x86 SHA-NI intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include @@ -558,32 +582,55 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m128i k = _mm_set1_epi32(2); return _mm_extract_epi32(_mm_sha256rnds2_epu32(i, i, k), 0); ]])], - [ AC_MSG_RESULT(yes); enable_shani=yes; AC_DEFINE(ENABLE_SHANI, 1, [Define this symbol to build code that uses SHA-NI intrinsics]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_x86_shani=yes; AC_DEFINE([ENABLE_X86_SHANI], [1], [Define this symbol to build code that uses x86 SHA-NI intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" # ARM -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto],[[ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"]],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crypto"], [], [$CXXFLAG_WERROR]) TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $ARM_CRC_CXXFLAGS" -AC_MSG_CHECKING(for ARM CRC32 intrinsics) +CXXFLAGS="$ARM_CRC_CXXFLAGS $CXXFLAGS" +AC_MSG_CHECKING([for ARMv8 CRC32 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include ]],[[ +#ifdef __aarch64__ __crc32cb(0, 0); __crc32ch(0, 0); __crc32cw(0, 0); __crc32cd(0, 0); vmull_p64(0, 0); +#else +#error "crc32c library does not support hardware acceleration on 32-bit ARM" +#endif ]])], - [ AC_MSG_RESULT(yes); enable_arm_crc=yes; ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_arm_crc=yes; ], + [ AC_MSG_RESULT([no])] +) +CXXFLAGS="$TEMP_CXXFLAGS" + +TEMP_CXXFLAGS="$CXXFLAGS" +CXXFLAGS="$ARM_SHANI_CXXFLAGS $CXXFLAGS" +AC_MSG_CHECKING([for ARMv8 SHA-NI intrinsics]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include + #include + ]],[[ + uint32x4_t a, b, c; + vsha256h2q_u32(a, b, c); + vsha256hq_u32(a, b, c); + vsha256su0q_u32(a, b); + vsha256su1q_u32(a, b, c); + ]])], + [ AC_MSG_RESULT([yes]); enable_arm_shani=yes; AC_DEFINE([ENABLE_ARM_SHANI], [1], [Define this symbol to build code that uses ARMv8 SHA-NI intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" fi -CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" +CORE_CPPFLAGS="$CORE_CPPFLAGS -DHAVE_BUILD_INFO" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], @@ -615,12 +662,24 @@ AC_ARG_ENABLE([util-util], [build_bitcoin_util=$enableval], [build_bitcoin_util=$build_bitcoin_utils]) +AC_ARG_ENABLE([experimental-util-chainstate], + [AS_HELP_STRING([--enable-experimental-util-chainstate], + [build experimental bitcoin-chainstate executable (default=no)])], + [build_bitcoin_chainstate=$enableval], + [build_bitcoin_chainstate=no]) + AC_ARG_WITH([libs], [AS_HELP_STRING([--with-libs], [build libraries (default=yes)])], [build_bitcoin_libs=$withval], [build_bitcoin_libs=yes]) +AC_ARG_WITH([experimental-kernel-lib], + [AS_HELP_STRING([--with-experimental-kernel-lib], + [build experimental bitcoinkernel library (default is to build if we're building libraries and the experimental build-chainstate executable)])], + [build_experimental_kernel_lib=$withval], + [build_experimental_kernel_lib=auto]) + AC_ARG_WITH([daemon], [AS_HELP_STRING([--with-daemon], [build bitcoind daemon (default=yes)])], @@ -630,36 +689,39 @@ AC_ARG_WITH([daemon], case $host in *mingw*) TARGET_OS=windows - AC_CHECK_LIB([kernel32], [GetModuleFileNameA],, AC_MSG_ERROR(libkernel32 missing)) - AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) - AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) - AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) - AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) - AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW],, AC_MSG_ERROR(libshell32 missing)) - AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) - AC_CHECK_LIB([ole32], [CoCreateInstance],, AC_MSG_ERROR(libole32 missing)) - AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) - AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) - AC_CHECK_LIB([advapi32], [CryptAcquireContextW],, AC_MSG_ERROR(libadvapi32 missing)) - AC_CHECK_LIB([ws2_32], [WSAStartup],, AC_MSG_ERROR(libws2_32 missing)) - AC_CHECK_LIB([shlwapi], [PathRemoveFileSpecW],, AC_MSG_ERROR(libshlwapi missing)) - AC_CHECK_LIB([iphlpapi], [GetAdaptersAddresses],, AC_MSG_ERROR(libiphlpapi missing)) + AC_CHECK_LIB([kernel32], [GetModuleFileNameA], [], [AC_MSG_ERROR([libkernel32 missing])]) + AC_CHECK_LIB([user32], [main], [], [AC_MSG_ERROR([libuser32 missing])]) + AC_CHECK_LIB([gdi32], [main], [], [AC_MSG_ERROR([libgdi32 missing])]) + AC_CHECK_LIB([comdlg32], [main], [], [AC_MSG_ERROR([libcomdlg32 missing])]) + AC_CHECK_LIB([winmm], [main], [], [AC_MSG_ERROR([libwinmm missing])]) + AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW], [], [AC_MSG_ERROR([libshell32 missing])]) + AC_CHECK_LIB([comctl32], [main], [], [AC_MSG_ERROR([libcomctl32 missing])]) + AC_CHECK_LIB([ole32], [CoCreateInstance], [], [AC_MSG_ERROR([libole32 missing])]) + AC_CHECK_LIB([oleaut32], [main], [], [AC_MSG_ERROR([liboleaut32 missing])]) + AC_CHECK_LIB([uuid], [main], [], [AC_MSG_ERROR([libuuid missing])]) + AC_CHECK_LIB([advapi32], [CryptAcquireContextW], [], [AC_MSG_ERROR([libadvapi32 missing])]) + AC_CHECK_LIB([ws2_32], [WSAStartup], [], [AC_MSG_ERROR([libws2_32 missing])]) + AC_CHECK_LIB([shlwapi], [PathRemoveFileSpecW], [], [AC_MSG_ERROR([libshlwapi missing])]) + AC_CHECK_LIB([iphlpapi], [GetAdaptersAddresses], [], [AC_MSG_ERROR([libiphlpapi missing])]) dnl -static is interpreted by libtool, where it has a different meaning. dnl In libtool-speak, it's -all-static. - AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) + AX_CHECK_LINK_FLAG([-static], [LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) - AC_PATH_PROG([MAKENSIS], [makensis], none) - if test x$MAKENSIS = xnone; then - AC_MSG_WARN("makensis not found. Cannot create installer.") + AC_PATH_PROG([MAKENSIS], [makensis], [none]) + if test "$MAKENSIS" = "none"; then + AC_MSG_WARN([makensis not found. Cannot create installer.]) fi - AC_PATH_TOOL(WINDRES, windres, none) - if test x$WINDRES = xnone; then - AC_MSG_ERROR("windres not found") + AC_PATH_TOOL([WINDRES], [windres], [none]) + if test "$WINDRES" = "none"; then + AC_MSG_ERROR([windres not found]) fi - CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" + dnl Prevent the definition of min/max macros. + dnl We always want to use the standard library. + CORE_CPPFLAGS="$CORE_CPPFLAGS -DNOMINMAX" dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -670,35 +732,67 @@ case $host in postdeps_CXX= dnl We require Windows 7 (NT 6.1) or later - AX_CHECK_LINK_FLAG([[-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1]],[LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"],,[[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) ;; *darwin*) TARGET_OS=darwin - if test x$cross_compiling != xyes; then + if test $cross_compiling != "yes"; then BUILD_OS=darwin - AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) - AC_CHECK_PROG([BREW],brew, brew) - if test x$BREW = xbrew; then + AC_CHECK_PROG([BREW], [brew], [brew]) + if test "$BREW" = "brew"; then dnl These Homebrew packages may be keg-only, meaning that they won't be found dnl in expected paths because they may conflict with system files. Ask dnl Homebrew where each one is located, then adjust paths accordingly. dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). - if test "x$use_bdb" != xno && $BREW list --versions berkeley-db4 >/dev/null && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then - bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) + dnl Homebrew may create symlinks in /usr/local/include for some packages. + dnl Because MacOS's clang internally adds "-I /usr/local/include" to its search + dnl paths, this will negate efforts to use -isystem for those packages, as they + dnl will be found first in /usr/local. Use the internal "-internal-isystem" + dnl option to system-ify all /usr/local/include paths without adding it to the list + dnl of search paths in case it's not already there. + if test "$suppress_external_warnings" != "no"; then + AX_CHECK_PREPROC_FLAG([-Xclang -internal-isystem/usr/local/include], [CORE_CPPFLAGS="$CORE_CPPFLAGS -Xclang -internal-isystem/usr/local/include"], [], [$CXXFLAG_WERROR]) + fi + + if test "$use_bdb" != "no" && $BREW list --versions berkeley-db@4 >/dev/null && test "$BDB_CFLAGS" = "" && test "$BDB_LIBS" = ""; then + bdb_prefix=$($BREW --prefix berkeley-db@4 2>/dev/null) dnl This must precede the call to BITCOIN_FIND_BDB48 below. BDB_CFLAGS="-I$bdb_prefix/include" BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" fi - if test "x$use_sqlite" != xno && $BREW list --versions sqlite3 >/dev/null; then - export PKG_CONFIG_PATH="$($BREW --prefix sqlite3 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" + if $BREW list --versions qt@5 >/dev/null; then + export PKG_CONFIG_PATH="$($BREW --prefix qt@5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi - if $BREW list --versions qt5 >/dev/null; then - export PKG_CONFIG_PATH="$($BREW --prefix qt5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" - fi + case $host in + *aarch64*) + dnl The preferred Homebrew prefix for Apple Silicon is /opt/homebrew. + dnl Therefore, as we do not use pkg-config to detect miniupnpc and libnatpmp + dnl packages, we should set the CPPFLAGS and LDFLAGS variables for them + dnl explicitly. + if test "$use_upnp" != "no" && $BREW list --versions miniupnpc >/dev/null; then + miniupnpc_prefix=$($BREW --prefix miniupnpc 2>/dev/null) + if test "$suppress_external_warnings" != "no"; then + MINIUPNPC_CPPFLAGS="-isystem $miniupnpc_prefix/include" + else + MINIUPNPC_CPPFLAGS="-I$miniupnpc_prefix/include" + fi + MINIUPNPC_LIBS="-L$miniupnpc_prefix/lib" + fi + if test "$use_natpmp" != "no" && $BREW list --versions libnatpmp >/dev/null; then + libnatpmp_prefix=$($BREW --prefix libnatpmp 2>/dev/null) + if test "$suppress_external_warnings" != "no"; then + NATPMP_CPPFLAGS="-isystem $libnatpmp_prefix/include" + else + NATPMP_CPPFLAGS="-I$libnatpmp_prefix/include" + fi + NATPMP_LIBS="-L$libnatpmp_prefix/lib" + fi + ;; + esac fi else case $build_os in @@ -706,14 +800,10 @@ case $host in BUILD_OS=darwin ;; *) - AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil) - AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) - AC_PATH_TOOL([OTOOL], [otool], otool) - AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs) - AC_PATH_PROGS([DMG], [dmg], dmg) - AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) - AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) - AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp) + AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil]) + AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], [install_name_tool]) + AC_PATH_TOOL([OTOOL], [otool], [otool]) + AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs]) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, @@ -724,8 +814,8 @@ case $host in esac fi - AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"],, [[$LDFLAG_WERROR]]) - CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" + AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) + CORE_CPPFLAGS="$CORE_CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" OBJCXXFLAGS="$CXXFLAGS" ;; *android*) @@ -741,10 +831,7 @@ case $host in *armv7a*) ANDROID_ARCH=armeabi-v7a ;; - *i686*) - ANDROID_ARCH=i686 - ;; - *) AC_MSG_ERROR("Could not determine Android arch") ;; + *) AC_MSG_ERROR([Could not determine Android arch, or it is unsupported]) ;; esac ;; *linux*) @@ -752,19 +839,19 @@ case $host in ;; esac -if test x$use_extended_functional_tests != xno; then +if test "$use_extended_functional_tests" != "no"; then AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi -if test x$use_lcov = xyes; then - if test x$LCOV = x; then - AC_MSG_ERROR("lcov testing requested but lcov not found") +if test "$use_lcov" = "yes"; then + if test "$LCOV" = ""; then + AC_MSG_ERROR([lcov testing requested but lcov not found]) fi - if test x$PYTHON = x; then - AC_MSG_ERROR("lcov testing requested but python not found") + if test "$PYTHON" = ""; then + AC_MSG_ERROR([lcov testing requested but python not found]) fi - if test x$GENHTML = x; then - AC_MSG_ERROR("lcov testing requested but genhtml not found") + if test "$GENHTML" = ""; then + AC_MSG_ERROR([lcov testing requested but genhtml not found]) fi AC_MSG_CHECKING([whether compiler is Clang]) @@ -776,13 +863,13 @@ if test x$use_lcov = xyes; then #endif ]])],[ AC_MSG_RESULT([yes]) - if test x$LLVM_COV = x; then + if test "$LLVM_COV" = ""; then AC_MSG_ERROR([lcov testing requested but llvm-cov not found]) fi COV_TOOL="$LLVM_COV gcov" ],[ AC_MSG_RESULT([no]) - if test x$GCOV = x; then + if test "$GCOV" = "x"; then AC_MSG_ERROR([lcov testing requested but gcov not found]) fi COV_TOOL="$GCOV" @@ -791,26 +878,32 @@ if test x$use_lcov = xyes; then AC_SUBST(COV_TOOL_WRAPPER, "cov_tool_wrapper.sh") LCOV="$LCOV --gcov-tool $(pwd)/$COV_TOOL_WRAPPER" - AX_CHECK_LINK_FLAG([[--coverage]], [LDFLAGS="$LDFLAGS --coverage"], - [AC_MSG_ERROR("lcov testing requested but --coverage linker flag does not work")]) - AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"], - [AC_MSG_ERROR("lcov testing requested but --coverage flag does not work")]) - CXXFLAGS="$CXXFLAGS -Og" + AX_CHECK_LINK_FLAG([--coverage], [CORE_LDFLAGS="$CORE_LDFLAGS --coverage"], + [AC_MSG_ERROR([lcov testing requested but --coverage linker flag does not work])]) + AX_CHECK_COMPILE_FLAG([--coverage],[CORE_CXXFLAGS="$CORE_CXXFLAGS --coverage"], + [AC_MSG_ERROR([lcov testing requested but --coverage flag does not work])]) + dnl If coverage is enabled, and the user hasn't overridden CXXFLAGS, clear + dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up + dnl with "--coverage -Og -O0 -g -O2". + if test "$CXXFLAGS_overridden" = "no"; then + CXXFLAGS="" + fi + CORE_CXXFLAGS="$CORE_CXXFLAGS -Og -O0" fi -if test x$use_lcov_branch != xno; then +if test "$use_lcov_branch" != "no"; then AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") fi -dnl Check for __int128 -AC_CHECK_TYPES([__int128]) - dnl Check for endianness AC_C_BIGENDIAN dnl Check for pthread compile/link requirements AX_PTHREAD +dnl Check if -latomic is required for +CHECK_ATOMIC + dnl The following macro will add the necessary defines to bitcoin-config.h, but dnl they also need to be passed down to any subprojects. Pull the results out of dnl the cache and add them to CPPFLAGS. @@ -818,65 +911,52 @@ AC_SYS_LARGEFILE dnl detect POSIX or GNU variant of strerror_r AC_FUNC_STRERROR_R -if test x$ac_cv_sys_file_offset_bits != x && - test x$ac_cv_sys_file_offset_bits != xno && - test x$ac_cv_sys_file_offset_bits != xunknown; then - CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" +if test "$ac_cv_sys_file_offset_bits" != "" && + test "$ac_cv_sys_file_offset_bits" != "no" && + test "$ac_cv_sys_file_offset_bits" != "unknown"; then + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" fi -if test x$ac_cv_sys_large_files != x && - test x$ac_cv_sys_large_files != xno && - test x$ac_cv_sys_large_files != xunknown; then - CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" +if test "$ac_cv_sys_large_files" != "" && + test "$ac_cv_sys_large_files" != "no" && + test "$ac_cv_sys_large_files" != "unknown"; then + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi -if test x$use_glibc_compat != xno; then - AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) - AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) - case $host in - powerpc64* | ppc64*) - AX_CHECK_LINK_FLAG([[-Wl,--no-tls-get-addr-optimize]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--no-tls-get-addr-optimize"]) - ;; - esac -else - AC_SEARCH_LIBS([clock_gettime],[rt]) -fi +AC_SEARCH_LIBS([clock_gettime],[rt]) -if test "x$enable_gprof" = xyes; then +if test "$enable_gprof" = "yes"; then dnl -pg is incompatible with -pie. Since hardening and profiling together doesn't make sense, dnl we simply make them mutually exclusive here. Additionally, hardened toolchains may force dnl -pie by default, in which case it needs to be turned off with -no-pie. - if test x$use_hardening = xyes; then - AC_MSG_ERROR(gprof profiling is not compatible with hardening. Reconfigure with --disable-hardening or --disable-gprof) + if test "$use_hardening" = "yes"; then + AC_MSG_ERROR([gprof profiling is not compatible with hardening. Reconfigure with --disable-hardening or --disable-gprof]) fi use_hardening=no AX_CHECK_COMPILE_FLAG([-pg],[GPROF_CXXFLAGS="-pg"], - [AC_MSG_ERROR(gprof profiling requested but not available)], [[$CXXFLAG_WERROR]]) + [AC_MSG_ERROR([gprof profiling requested but not available])], [$CXXFLAG_WERROR]) - AX_CHECK_LINK_FLAG([[-no-pie]], [GPROF_LDFLAGS="-no-pie"]) - AX_CHECK_LINK_FLAG([[-pg]],[GPROF_LDFLAGS="$GPROF_LDFLAGS -pg"], - [AC_MSG_ERROR(gprof profiling requested but not available)], [[$GPROF_LDFLAGS]]) + AX_CHECK_LINK_FLAG([-no-pie], [GPROF_LDFLAGS="-no-pie"]) + AX_CHECK_LINK_FLAG([-pg], [GPROF_LDFLAGS="$GPROF_LDFLAGS -pg"], + [AC_MSG_ERROR([gprof profiling requested but not available])], [$GPROF_LDFLAGS]) fi -if test x$TARGET_OS != xwindows; then +if test "$TARGET_OS" != "windows"; then dnl All windows code is PIC, forcing it on just adds useless compile warnings - AX_CHECK_COMPILE_FLAG([-fPIC],[PIC_FLAGS="-fPIC"]) + AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi dnl All versions of gcc that we commonly use for building are subject to bug dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) -AX_CHECK_COMPILE_FLAG([-fstack-reuse=none],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) -if test x$use_hardening != xno; then +AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) +if test "$use_hardening" != "no"; then use_hardening=yes - AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) - AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) + AX_CHECK_COMPILE_FLAG([-Wstack-protector], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) + AX_CHECK_COMPILE_FLAG([-fstack-protector-all], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) - dnl -fcf-protection used with Clang 7 causes ld to emit warnings: - dnl ld: error: ... - dnl Use CHECK_LINK_FLAG & --fatal-warnings to ensure we won't use the flag in this case. - AX_CHECK_LINK_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"],, [[$LDFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-fcf-protection=full], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"]) case $host in *mingw*) @@ -892,7 +972,7 @@ if test x$use_hardening != xno; then dnl When enable_debug is yes, all optimizations are disabled. dnl However, FORTIFY_SOURCE requires that there is some level of optimization, otherwise it does nothing and just creates a compiler warning. dnl Since FORTIFY_SOURCE is a no-op without optimizations, do not enable it when enable_debug is yes. - if test x$enable_debug != xyes; then + if test "$enable_debug" != "yes"; then AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=2],[ AX_CHECK_PREPROC_FLAG([-U_FORTIFY_SOURCE],[ HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -U_FORTIFY_SOURCE" @@ -901,18 +981,18 @@ if test x$use_hardening != xno; then ]) fi - AX_CHECK_LINK_FLAG([[-Wl,--enable-reloc-section]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--enable-reloc-section"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-z,relro]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-z,now]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-z,separate-code]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,separate-code"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-fPIE -pie]], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"],, [[$CXXFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-Wl,--enable-reloc-section], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--enable-reloc-section"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--dynamicbase], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--nxcompat], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--high-entropy-va], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-z,relro], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-z,now], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-z,separate-code], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,separate-code"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-fPIE -pie], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"], [], [$CXXFLAG_WERROR]) case $host in *mingw*) - AC_CHECK_LIB([ssp], [main],, AC_MSG_ERROR(libssp missing)) + AC_CHECK_LIB([ssp], [main], [], [AC_MSG_ERROR([libssp missing])]) ;; esac fi @@ -920,19 +1000,18 @@ fi dnl These flags are specific to ld64, and may cause issues with other linkers. dnl For example: GNU ld will interpret -dead_strip as -de and then try and use dnl "ad_strip" as the symbol for the entry point. -if test x$TARGET_OS = xdarwin; then - AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-dead_strip_dylibs]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-bind_at_load]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"],, [[$LDFLAG_WERROR]]) +if test "$TARGET_OS" = "darwin"; then + AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) fi -AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) +AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h unistd.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, [#include #include ] ) -AC_CHECK_DECLS([strnlen]) dnl These are used for daemonization in bitcoind AC_CHECK_DECLS([fork]) @@ -940,52 +1019,54 @@ AC_CHECK_DECLS([setsid]) AC_CHECK_DECLS([pipe2]) +AC_CHECK_FUNCS([timingsafe_bcmp]) + AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,, - [#if HAVE_ENDIAN_H + [#if HAVE_ENDIAN_H #include #elif HAVE_SYS_ENDIAN_H #include #endif]) AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, - [#if HAVE_BYTESWAP_H + [#if HAVE_BYTESWAP_H #include #endif]) -AC_MSG_CHECKING(for __builtin_clzl) +AC_MSG_CHECKING([for __builtin_clzl]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ (void) __builtin_clzl(0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); have_clzl=yes; AC_DEFINE([HAVE_BUILTIN_CLZL], [1], [Define this symbol if you have __builtin_clzl])], + [ AC_MSG_RESULT([no]); have_clzl=no;] ) -AC_MSG_CHECKING(for __builtin_clzll) +AC_MSG_CHECKING([for __builtin_clzll]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ (void) __builtin_clzll(0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); have_clzll=yes; AC_DEFINE([HAVE_BUILTIN_CLZLL], [1], [Define this symbol if you have __builtin_clzll])], + [ AC_MSG_RESULT([no]); have_clzll=no;] ) dnl Check for malloc_info (for memory statistics information in getmemoryinfo) -AC_MSG_CHECKING(for getmemoryinfo) +AC_MSG_CHECKING([for getmemoryinfo]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ int f = malloc_info(0, NULL); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOC_INFO, 1,[Define this symbol if you have malloc_info]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_MALLOC_INFO], [1], [Define this symbol if you have malloc_info]) ], + [ AC_MSG_RESULT([no])] ) dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas) -AC_MSG_CHECKING(for mallopt M_ARENA_MAX) +AC_MSG_CHECKING([for mallopt M_ARENA_MAX]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ mallopt(M_ARENA_MAX, 1); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOPT_ARENA_MAX, 1,[Define this symbol if you have mallopt with M_ARENA_MAX]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_MALLOPT_ARENA_MAX], [1], [Define this symbol if you have mallopt with M_ARENA_MAX]) ], + [ AC_MSG_RESULT([no])] ) dnl Check for posix_fallocate -AC_MSG_CHECKING(for posix_fallocate) +AC_MSG_CHECKING([for posix_fallocate]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ // same as in src/util/system.cpp #ifdef __linux__ @@ -996,8 +1077,8 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #endif // __linux__ #include ]], [[ int f = posix_fallocate(0, 0, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_POSIX_FALLOCATE, 1,[Define this symbol if you have posix_fallocate]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_POSIX_FALLOCATE], [1], [Define this symbol if you have posix_fallocate]) ], + [ AC_MSG_RESULT([no])] ) AC_MSG_CHECKING([for default visibility attribute]) @@ -1006,12 +1087,12 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([ int main(){} ])], [ - AC_DEFINE(HAVE_DEFAULT_VISIBILITY_ATTRIBUTE,1,[Define if the visibility attribute is supported.]) - AC_MSG_RESULT(yes) + AC_DEFINE([HAVE_DEFAULT_VISIBILITY_ATTRIBUTE], [1], [Define if the visibility attribute is supported.]) + AC_MSG_RESULT([yes]) ], [ - AC_MSG_RESULT(no) - if test x$use_reduce_exports = xyes; then + AC_MSG_RESULT([no]) + if test "$use_reduce_exports" = "yes"; then AC_MSG_ERROR([Cannot find a working visibility attribute. Use --disable-reduce-exports.]) fi ] @@ -1023,16 +1104,13 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([ int main(){} ])], [ - AC_DEFINE(HAVE_DLLEXPORT_ATTRIBUTE,1,[Define if the dllexport attribute is supported.]) - AC_MSG_RESULT(yes) + AC_DEFINE([HAVE_DLLEXPORT_ATTRIBUTE], [1], [Define if the dllexport attribute is supported.]) + AC_MSG_RESULT([yes]) ], - [AC_MSG_RESULT(no)] + [AC_MSG_RESULT([no])] ) -dnl thread_local is currently disabled when building with glibc back compat. -dnl Our minimum supported glibc is 2.17, however support for thread_local -dnl did not arrive in glibc until 2.18. -if test "x$use_thread_local" = xyes || { test "x$use_thread_local" = xauto && test "x$use_glibc_compat" = xno; }; then +if test "$use_thread_local" = "yes" || test "$use_thread_local" = "auto"; then TEMP_LDFLAGS="$LDFLAGS" LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" AC_MSG_CHECKING([for thread_local support]) @@ -1051,21 +1129,21 @@ if test "x$use_thread_local" = xyes || { test "x$use_thread_local" = xauto && te dnl mingw32's implementation of thread_local has also been shown to behave dnl erroneously under concurrent usage; see: dnl https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) ;; *freebsd*) dnl FreeBSD's implementation of thread_local is also buggy (per dnl https://groups.google.com/d/msg/bsdmailinglist/22ncTZAbDp4/Dii_pII5AwAJ) - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) ;; *) - AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) - AC_MSG_RESULT(yes) + AC_DEFINE([HAVE_THREAD_LOCAL], [1], [Define if thread_local is supported.]) + AC_MSG_RESULT([yes]) ;; esac ], [ - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) ] ) LDFLAGS="$TEMP_LDFLAGS" @@ -1073,57 +1151,50 @@ fi dnl check for gmtime_r(), fallback to gmtime_s() if that is unavailable dnl fail if neither are available. -AC_MSG_CHECKING(for gmtime_r) +AC_MSG_CHECKING([for gmtime_r]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ gmtime_r((const time_t *) nullptr, (struct tm *) nullptr); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GMTIME_R, 1, [Define this symbol if gmtime_r is available]) ], - [ AC_MSG_RESULT(no); - AC_MSG_CHECKING(for gmtime_s); + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GMTIME_R], [1], [Define this symbol if gmtime_r is available]) ], + [ AC_MSG_RESULT([no]); + AC_MSG_CHECKING([for gmtime_s]); AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ gmtime_s((struct tm *) nullptr, (const time_t *) nullptr); ]])], - [ AC_MSG_RESULT(yes)], - [ AC_MSG_RESULT(no); AC_MSG_ERROR(Both gmtime_r and gmtime_s are unavailable) ] + [ AC_MSG_RESULT([yes])], + [ AC_MSG_RESULT([no]); AC_MSG_ERROR([Both gmtime_r and gmtime_s are unavailable]) ] ) ] ) dnl Check for different ways of gathering OS randomness -AC_MSG_CHECKING(for Linux getrandom syscall) +AC_MSG_CHECKING([for Linux getrandom syscall]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include #include ]], [[ syscall(SYS_getrandom, nullptr, 32, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYS_GETRANDOM, 1,[Define this symbol if the Linux getrandom system call is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYS_GETRANDOM], [1], [Define this symbol if the Linux getrandom system call is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for getentropy) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], - [[ getentropy(nullptr, 32) ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY, 1,[Define this symbol if the BSD getentropy system call is available]) ], - [ AC_MSG_RESULT(no)] -) - -AC_MSG_CHECKING(for getentropy via random.h) +AC_MSG_CHECKING([for getentropy via random.h]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ getentropy(nullptr, 32) ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY_RAND, 1,[Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETENTROPY_RAND], [1], [Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for sysctl) +AC_MSG_CHECKING([for sysctl]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ #ifdef __linux__ #error "Don't use sysctl on Linux, it's deprecated even when it works" #endif sysctl(nullptr, 2, nullptr, nullptr, nullptr, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL, 1,[Define this symbol if the BSD sysctl() is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYSCTL], [1], [Define this symbol if the BSD sysctl() is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for sysctl KERN_ARND) +AC_MSG_CHECKING([for sysctl KERN_ARND]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ #ifdef __linux__ @@ -1131,116 +1202,96 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #endif static int name[2] = {CTL_KERN, KERN_ARND}; sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL_ARND, 1,[Define this symbol if the BSD sysctl(KERN_ARND) is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYSCTL_ARND], [1], [Define this symbol if the BSD sysctl(KERN_ARND) is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for if type char equals int8_t) +AC_MSG_CHECKING([for if type char equals int8_t]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ static_assert(std::is_same::value, ""); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(CHAR_EQUALS_INT8, 1,[Define this symbol if type char equals int8_t]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([CHAR_EQUALS_INT8], [1], [Define this symbol if type char equals int8_t]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for fdatasync) +AC_MSG_CHECKING([for fdatasync]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ fdatasync(0); ]])], - [ AC_MSG_RESULT(yes); HAVE_FDATASYNC=1 ], - [ AC_MSG_RESULT(no); HAVE_FDATASYNC=0 ] + [ AC_MSG_RESULT([yes]); HAVE_FDATASYNC=1 ], + [ AC_MSG_RESULT([no]); HAVE_FDATASYNC=0 ] ) AC_DEFINE_UNQUOTED([HAVE_FDATASYNC], [$HAVE_FDATASYNC], [Define to 1 if fdatasync is available.]) -AC_MSG_CHECKING(for F_FULLFSYNC) +AC_MSG_CHECKING([for F_FULLFSYNC]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ fcntl(0, F_FULLFSYNC, 0); ]])], - [ AC_MSG_RESULT(yes); HAVE_FULLFSYNC=1 ], - [ AC_MSG_RESULT(no); HAVE_FULLFSYNC=0 ] + [ AC_MSG_RESULT([yes]); HAVE_FULLFSYNC=1 ], + [ AC_MSG_RESULT([no]); HAVE_FULLFSYNC=0 ] ) -AC_MSG_CHECKING(for O_CLOEXEC) +AC_MSG_CHECKING([for O_CLOEXEC]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ open("", O_CLOEXEC); ]])], - [ AC_MSG_RESULT(yes); HAVE_O_CLOEXEC=1 ], - [ AC_MSG_RESULT(no); HAVE_O_CLOEXEC=0 ] + [ AC_MSG_RESULT([yes]); HAVE_O_CLOEXEC=1 ], + [ AC_MSG_RESULT([no]); HAVE_O_CLOEXEC=0 ] ) AC_DEFINE_UNQUOTED([HAVE_O_CLOEXEC], [$HAVE_O_CLOEXEC], [Define to 1 if O_CLOEXEC flag is available.]) dnl crc32c platform checks -AC_MSG_CHECKING(for __builtin_prefetch) +AC_MSG_CHECKING([for __builtin_prefetch]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ char data = 0; const char* address = &data; __builtin_prefetch(address, 0, 0); ]])], - [ AC_MSG_RESULT(yes); HAVE_BUILTIN_PREFETCH=1 ], - [ AC_MSG_RESULT(no); HAVE_BUILTIN_PREFETCH=0 ] + [ AC_MSG_RESULT([yes]); HAVE_BUILTIN_PREFETCH=1 ], + [ AC_MSG_RESULT([no]); HAVE_BUILTIN_PREFETCH=0 ] ) -AC_MSG_CHECKING(for _mm_prefetch) +AC_MSG_CHECKING([for _mm_prefetch]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ char data = 0; const char* address = &data; _mm_prefetch(address, _MM_HINT_NTA); ]])], - [ AC_MSG_RESULT(yes); HAVE_MM_PREFETCH=1 ], - [ AC_MSG_RESULT(no); HAVE_MM_PREFETCH=0 ] + [ AC_MSG_RESULT([yes]); HAVE_MM_PREFETCH=1 ], + [ AC_MSG_RESULT([no]); HAVE_MM_PREFETCH=0 ] ) -AC_MSG_CHECKING(for strong getauxval support in the system headers) +AC_MSG_CHECKING([for strong getauxval support in the system headers]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include - #include #include ]], [[ getauxval(AT_HWCAP); ]])], - [ AC_MSG_RESULT(yes); HAVE_STRONG_GETAUXVAL=1; AC_DEFINE(HAVE_STRONG_GETAUXVAL, 1, [Define this symbol to build code that uses getauxval)]) ], - [ AC_MSG_RESULT(no); HAVE_STRONG_GETAUXVAL=0 ] -) - -AC_MSG_CHECKING(for weak getauxval support in the compiler) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #ifdef __linux__ - unsigned long getauxval(unsigned long type) __attribute__((weak)); - #define AT_HWCAP 16 - #endif - ]], [[ - getauxval(AT_HWCAP); - ]])], - [ AC_MSG_RESULT(yes); HAVE_WEAK_GETAUXVAL=1; AC_DEFINE(HAVE_WEAK_GETAUXVAL, 1, [Define this symbol to build code that uses getauxval (weak linking)]) ], - [ AC_MSG_RESULT(no); HAVE_WEAK_GETAUXVAL=0 ] + [ AC_MSG_RESULT([yes]); HAVE_STRONG_GETAUXVAL=1; AC_DEFINE([HAVE_STRONG_GETAUXVAL], [1], [Define this symbol to build code that uses getauxval)]) ], + [ AC_MSG_RESULT([no]); HAVE_STRONG_GETAUXVAL=0 ] ) +have_any_system=no AC_MSG_CHECKING([for std::system]) AC_LINK_IFELSE( [ AC_LANG_PROGRAM( [[ #include ]], [[ int nErr = std::system(""); ]] )], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_STD__SYSTEM, 1, Define to 1 if std::system is available.)], - [ AC_MSG_RESULT(no) ] + [ AC_MSG_RESULT([yes]); have_any_system=yes], + [ AC_MSG_RESULT([no]) ] ) AC_MSG_CHECKING([for ::_wsystem]) AC_LINK_IFELSE( [ AC_LANG_PROGRAM( - [[ ]], - [[ int nErr = ::_wsystem(""); ]] + [[ #include ]], + [[ int nErr = ::_wsystem(NULL); ]] )], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_WSYSTEM, 1, Define to 1 if ::wsystem is available.)], - [ AC_MSG_RESULT(no) ] + [ AC_MSG_RESULT([yes]); have_any_system=yes], + [ AC_MSG_RESULT([no]) ] ) -AC_DEFINE([HAVE_SYSTEM], [HAVE_STD__SYSTEM || HAVE_WSYSTEM], [std::system or ::wsystem]) - -LEVELDB_CPPFLAGS= -LIBLEVELDB= -LIBMEMENV= -AM_CONDITIONAL([EMBEDDED_LEVELDB],[true]) -AC_SUBST(LEVELDB_CPPFLAGS) -AC_SUBST(LIBLEVELDB) -AC_SUBST(LIBMEMENV) +if test "$have_any_system" != "no"; then + AC_DEFINE([HAVE_SYSTEM], [1], [Define to 1 if std::system or ::wsystem is available.]) +fi dnl SUPPRESSED_CPPFLAGS=SUPPRESS_WARNINGS([$SOME_CPPFLAGS]) dnl Replace -I with -isystem in $SOME_CPPFLAGS to suppress warnings from @@ -1252,86 +1303,88 @@ dnl Do not change "-I/usr/include" to "-isystem /usr/include" because that dnl is not necessary (/usr/include is already a system directory) and because dnl it would break GCC's #include_next. AC_DEFUN([SUPPRESS_WARNINGS], - [$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include([/ ]|$);-I/usr/include\1;g')]) + [[$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include/*( |$);-I/usr/include\1;g')]]) dnl enable-fuzz should disable all other targets -if test "x$enable_fuzz" = "xyes"; then - AC_MSG_WARN(enable-fuzz will disable all other targets and force --enable-fuzz-binary=yes) +if test "$enable_fuzz" = "yes"; then + AC_MSG_WARN([enable-fuzz will disable all other targets and force --enable-fuzz-binary=yes]) build_bitcoin_utils=no build_bitcoin_cli=no build_bitcoin_tx=no build_bitcoin_util=no + build_bitcoin_chainstate=no build_bitcoin_wallet=no build_bitcoind=no build_bitcoin_libs=no bitcoin_enable_qt=no bitcoin_enable_qt_test=no bitcoin_enable_qt_dbus=no - enable_wallet=no use_bench=no + use_tests=no use_external_signer=no use_upnp=no use_natpmp=no use_zmq=no enable_fuzz_binary=yes - AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) - - AC_MSG_CHECKING([whether main function is needed for fuzz binary]) - AX_CHECK_LINK_FLAG( - [[-fsanitize=$use_sanitizers]], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([yes]) - CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], - [], - [AC_LANG_PROGRAM([[ - #include - #include - extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; } - /* comment to remove the main function ... - ]],[[ - */ int not_main() { - ]])]) + AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) else BITCOIN_QT_INIT dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus - BITCOIN_QT_CONFIGURE([5.9.5]) + BITCOIN_QT_CONFIGURE([5.11.3]) dnl Keep a copy of the original $QT_INCLUDES and use it when invoking qt's moc QT_INCLUDES_UNSUPPRESSED=$QT_INCLUDES - if test x$suppress_external_warnings != xno ; then + if test "$suppress_external_warnings" != "no" ; then QT_INCLUDES=SUPPRESS_WARNINGS($QT_INCLUDES) QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES) QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) fi +fi + +if test "$enable_fuzz_binary" = "yes"; then + AC_MSG_CHECKING([whether main function is needed for fuzz binary]) + AX_CHECK_LINK_FLAG( + [], + [AC_MSG_RESULT([no])], + [AC_MSG_RESULT([yes]); CORE_CPPFLAGS="$CORE_CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], + [$SANITIZER_LDFLAGS], + [AC_LANG_PROGRAM([[ + #include + #include + extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { return 0; } + /* comment to remove the main function ... + ]],[[ + */ int not_main() { + ]])]) - CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION" + CHECK_RUNTIME_LIB fi -if test x$enable_wallet != xno; then +if test "$enable_wallet" != "no"; then dnl Check for libdb_cxx only if wallet enabled - if test "x$use_bdb" != "xno"; then + if test "$use_bdb" != "no"; then BITCOIN_FIND_BDB48 - if test x$suppress_external_warnings != xno ; then + if test "$suppress_external_warnings" != "no" ; then BDB_CPPFLAGS=SUPPRESS_WARNINGS($BDB_CPPFLAGS) fi fi dnl Check for sqlite3 - if test "x$use_sqlite" != "xno"; then + if test "$use_sqlite" != "no"; then PKG_CHECK_MODULES([SQLITE], [sqlite3 >= 3.7.17], [have_sqlite=yes], [have_sqlite=no]) fi AC_MSG_CHECKING([whether to build wallet with support for sqlite]) - if test "x$use_sqlite" = "xno"; then + if test "$use_sqlite" = "no"; then use_sqlite=no - elif test "x$have_sqlite" = "xno"; then - if test "x$use_sqlite" = "xyes"; then + elif test "$have_sqlite" = "no"; then + if test "$use_sqlite" = "yes"; then AC_MSG_ERROR([sqlite support requested but cannot be built. Use --without-sqlite]) fi use_sqlite=no else - if test x$use_sqlite != xno; then + if test "$use_sqlite" != "no"; then AC_DEFINE([USE_SQLITE],[1],[Define if sqlite support should be compiled in]) use_sqlite=yes fi @@ -1339,150 +1392,236 @@ if test x$enable_wallet != xno; then AC_MSG_RESULT([$use_sqlite]) dnl Disable wallet if both --without-bdb and --without-sqlite - if test "x$use_bdb$use_sqlite" = "xnono"; then - if test "x$enable_wallet" = "xyes"; then + if test "$use_bdb$use_sqlite" = "nono"; then + if test "$enable_wallet" = "yes"; then AC_MSG_ERROR([wallet functionality requested but no BDB or SQLite support available.]) fi enable_wallet=no fi fi -if test x$use_ebpf != xno; then - AC_MSG_CHECKING([whether eBPF tracepoints are supported]) +if test "$use_usdt" != "no"; then + AC_MSG_CHECKING([whether Userspace, Statically Defined Tracing tracepoints are supported]) AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [#include ], [DTRACE_PROBE("context", "event");] )], - [AC_MSG_RESULT(yes); have_sdt=yes; AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable eBPF user static defined tracepoints])], - [AC_MSG_RESULT(no); have_sdt=no;] + [AC_MSG_RESULT([yes]); AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable tracepoints for Userspace, Statically Defined Tracing])], + [AC_MSG_RESULT([no]); use_usdt=no;] ) fi +AM_CONDITIONAL([ENABLE_USDT_TRACEPOINTS], [test "$use_usdt" = "yes"]) + +if test "$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nononono"; then + use_upnp=no + use_natpmp=no + use_zmq=no +fi dnl Check for libminiupnpc (optional) -if test x$use_upnp != xno; then +if test "$use_upnp" != "no"; then + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $MINIUPNPC_CPPFLAGS" AC_CHECK_HEADERS( [miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], - [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])], + [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS])], [have_miniupnpc=no] ) -dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility -dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. -if test x$have_miniupnpc != xno; then - AC_MSG_CHECKING([whether miniUPnPc API version is supported]) - AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ - @%:@include - ]], [[ - #if MINIUPNPC_API_VERSION >= 10 - // Everything is okay - #else - # error miniUPnPc API version is too old - #endif - ]])],[ - AC_MSG_RESULT(yes) - ],[ - AC_MSG_RESULT(no) - AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.]) - have_miniupnpc=no - ]) -fi + dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility + dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. + if test "$have_miniupnpc" != "no"; then + AC_MSG_CHECKING([whether miniUPnPc API version is supported]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if MINIUPNPC_API_VERSION >= 10 + // Everything is okay + #else + # error miniUPnPc API version is too old + #endif + ]])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.]) + have_miniupnpc=no + ]) + fi + CPPFLAGS="$TEMP_CPPFLAGS" fi dnl Check for libnatpmp (optional). -if test "x$use_natpmp" != xno; then +if test "$use_natpmp" != "no"; then + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $NATPMP_CPPFLAGS" AC_CHECK_HEADERS([natpmp.h], - [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS=-lnatpmp], [have_natpmp=no])], + [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS="$NATPMP_LIBS -lnatpmp"], [have_natpmp=no], [$NATPMP_LIBS])], [have_natpmp=no]) + CPPFLAGS="$TEMP_CPPFLAGS" fi -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" = "nonononononono"; then use_boost=no else use_boost=yes fi -if test x$use_boost = xyes; then +if test "$use_boost" = "yes"; then dnl Check for Boost headers AX_BOOST_BASE([1.64.0],[],[AC_MSG_ERROR([Boost is not available!])]) - if test x$want_boost = xno; then - AC_MSG_ERROR([[only libbitcoinconsensus can be built without boost]]) + if test "$want_boost" = "no"; then + AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost]) fi - AX_BOOST_SYSTEM - AX_BOOST_FILESYSTEM - if test x$suppress_external_warnings != xno; then - BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) + dnl we don't use multi_index serialization + BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_DISABLE_SERIALIZATION" + + dnl Prevent use of std::unary_function, which was removed in C++17, + dnl and will generate warnings with newer compilers. + dnl See: https://github.com/boostorg/container_hash/issues/22. + BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE" + + if test "$enable_debug" = "yes" || test "$enable_fuzz" = "yes"; then + BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE" fi - BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB" + if test "$suppress_external_warnings" != "no"; then + BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) + fi fi -if test "x$use_external_signer" != xno; then - AC_DEFINE([ENABLE_EXTERNAL_SIGNER],,[define if external signer support is enabled]) +if test "$use_external_signer" != "no"; then + case $host in + *mingw*) + dnl Boost Process uses Boost Filesystem when targeting Windows. Also, + dnl since Boost 1.71.0, Process does not work with mingw-w64 without + dnl workarounds. See 67669ab425b52a2b6be3d2f3b3b7e3939b676a2c. + if test "$use_external_signer" = "yes"; then + AC_MSG_ERROR([External signing is not supported on Windows]) + fi + use_external_signer="no"; + ;; + *) + AC_MSG_CHECKING([whether Boost.Process can be used]) + TEMP_CXXFLAGS="$CXXFLAGS" + dnl Boost 1.78 requires the following workaround. + dnl See: https://github.com/boostorg/process/issues/235 + CXXFLAGS="$CXXFLAGS -Wno-error=narrowing" + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + TEMP_LDFLAGS="$LDFLAGS" + dnl Boost 1.73 and older require the following workaround. + LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]])], + [have_boost_process="yes"], + [have_boost_process="no"]) + LDFLAGS="$TEMP_LDFLAGS" + CPPFLAGS="$TEMP_CPPFLAGS" + CXXFLAGS="$TEMP_CXXFLAGS" + AC_MSG_RESULT([$have_boost_process]) + if test "$have_boost_process" = "yes"; then + use_external_signer="yes" + AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) + else + if test "$use_external_signer" = "yes"; then + AC_MSG_ERROR([External signing is not supported for this Boost version]) + fi + use_external_signer="no"; + fi + ;; + esac +fi +AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) + +dnl Do not compile with syscall sandbox support when compiling under the sanitizers. +dnl The sanitizers introduce use of syscalls that are not typically used in bitcoind +dnl (such as execve when the sanitizers execute llvm-symbolizer). +if test "$use_sanitizers" != ""; then + AC_MSG_WARN([Specifying --with-sanitizers forces --without-seccomp since the sanitizers introduce use of syscalls not allowed by the bitcoind syscall sandbox (-sandbox=).]) + seccomp_found=no +fi +if test "$seccomp_found" != "no"; then + AC_MSG_CHECKING([for seccomp-bpf (Linux x86-64)]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if !defined(__x86_64__) + # error Syscall sandbox is an experimental feature currently available only under Linux x86-64. + #endif + ]])],[ + AC_MSG_RESULT([yes]) + seccomp_found="yes" + AC_DEFINE([USE_SYSCALL_SANDBOX], [1], [Define this symbol to build with syscall sandbox support.]) + ],[ + AC_MSG_RESULT([no]) + seccomp_found="no" + ]) fi -AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "x$use_external_signer" = "xyes"]) +dnl Currently only enable -sandbox= feature if seccomp is found. +dnl In the future, sandboxing could be also be supported with other +dnl sandboxing mechanisms besides seccomp. +use_syscall_sandbox=$seccomp_found +AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "$use_syscall_sandbox" != "no"]) dnl Check for reduced exports -if test x$use_reduce_exports = xyes; then - AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], - [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])],[[$CXXFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]],[RELDFLAGS="-Wl,--exclude-libs,ALL"],,[[$LDFLAG_WERROR]]) +if test "$use_reduce_exports" = "yes"; then + AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fvisibility=hidden"], + [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])], [$CXXFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--exclude-libs,ALL], [RELDFLAGS="-Wl,--exclude-libs,ALL"], [], [$LDFLAG_WERROR]) fi -if test x$use_tests = xyes; then - - if test x$HEXDUMP = x; then - AC_MSG_ERROR(hexdump is required for tests) - fi - - if test x$use_boost = xyes; then - - AX_BOOST_UNIT_TEST_FRAMEWORK - - dnl Determine if -DBOOST_TEST_DYN_LINK is needed - AC_MSG_CHECKING([for dynamic linked boost test]) - TEMP_LIBS="$LIBS" - LIBS="$LIBS $BOOST_LDFLAGS $BOOST_UNIT_TEST_FRAMEWORK_LIB" - TEMP_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - AC_LINK_IFELSE([AC_LANG_SOURCE([ - #define BOOST_TEST_DYN_LINK - #define BOOST_TEST_MAIN - #include - - ])], - [AC_MSG_RESULT(yes)] - [TESTDEFS="$TESTDEFS -DBOOST_TEST_DYN_LINK"], - [AC_MSG_RESULT(no)]) - LIBS="$TEMP_LIBS" - CPPFLAGS="$TEMP_CPPFLAGS" +if test "$use_tests" = "yes"; then + if test "$HEXDUMP" = ""; then + AC_MSG_ERROR([hexdump is required for tests]) fi fi dnl libevent check -if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])]) - if test x$TARGET_OS != xwindows; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])]) +use_libevent=no +if test "$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_tests$use_bench" != "nononononono"; then + PKG_CHECK_MODULES([EVENT], [libevent >= 2.1.8], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.1.8 or greater not found.])]) + if test "$TARGET_OS" != "windows"; then + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.1.8], [], [AC_MSG_ERROR([libevent_pthreads version 2.1.8 or greater not found.])]) fi - if test x$suppress_external_warnings != xno; then + if test "$suppress_external_warnings" != "no"; then EVENT_CFLAGS=SUPPRESS_WARNINGS($EVENT_CFLAGS) fi fi +if test x$use_libevent = xyes; then + TEMP_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $EVENT_CFLAGS" + AC_MSG_CHECKING([if evhttp_connection_get_peer expects const char**]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include + #include + ]], [[ + evhttp_connection *conn = (evhttp_connection *)1; + const char *host; + uint16_t port; + + evhttp_connection_get_peer(conn, &host, &port); + ]])], + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR], [1], [Define this symbol if evhttp_connection_get_peer expects const char**]) ], + [ AC_MSG_RESULT([no]) ] + ) + CXXFLAGS="$TEMP_CXXFLAGS" +fi + dnl QR Code encoding library check -if test "x$use_qr" != xno; then +if test "$use_qr" != "no"; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi dnl ZMQ check -if test "x$use_zmq" = xyes; then +if test "$use_zmq" = "yes"; then PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) @@ -1492,7 +1631,7 @@ else AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) fi -if test "x$use_zmq" = xyes; then +if test "$use_zmq" = "yes"; then dnl Assume libzmq was built for static linking case $host in *mingw*) @@ -1501,110 +1640,93 @@ if test "x$use_zmq" = xyes; then esac fi -dnl univalue check - -need_bundled_univalue=yes -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnononononononono; then - need_bundled_univalue=no -else - if test x$system_univalue != xno; then - PKG_CHECK_MODULES([UNIVALUE], [libunivalue >= 1.0.4], [found_univalue=yes], [found_univalue=no]) - if test x$found_univalue = xyes; then - system_univalue=yes - need_bundled_univalue=no - elif test x$system_univalue = xyes; then - AC_MSG_ERROR([univalue not found]) - else - system_univalue=no - fi - fi - - if test x$need_bundled_univalue = xyes; then - UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' - UNIVALUE_LIBS='univalue/libunivalue.la' - fi -fi - -AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) -AC_SUBST(UNIVALUE_CFLAGS) -AC_SUBST(UNIVALUE_LIBS) - dnl libmultiprocess library check libmultiprocess_found=no -if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then - m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ +if test "$with_libmultiprocess" = "yes" || test "$with_libmultiprocess" = "auto"; then + PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ libmultiprocess_found=yes; libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; - ], [true])]) -elif test "x$with_libmultiprocess" != xno; then + ], [true]) +elif test "$with_libmultiprocess" != "no"; then AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) fi -AC_SUBST(LIBMULTIPROCESS_CFLAGS) -AC_SUBST(LIBMULTIPROCESS_LIBS) dnl Enable multiprocess check -if test "x$enable_multiprocess" = xyes; then - if test "x$libmultiprocess_found" != xyes; then +if test "$enable_multiprocess" = "yes"; then + if test "$libmultiprocess_found" != "yes"; then AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.]) fi build_multiprocess=yes -elif test "x$enable_multiprocess" = xauto; then +elif test "$enable_multiprocess" = "auto"; then build_multiprocess=$libmultiprocess_found else build_multiprocess=no fi -AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes]) -AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes]) -AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes]) +AM_CONDITIONAL([BUILD_MULTIPROCESS], [test "$build_multiprocess" = "yes"]) +AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "$build_multiprocess" = "yes"]) +AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "$build_multiprocess" = "yes"]) dnl codegen tools check -if test x$build_multiprocess != xno; then - if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then +if test "$build_multiprocess" != "no"; then + if test "$with_mpgen" = "yes" || test "$with_mpgen" = "auto"; then MPGEN_PREFIX="$libmultiprocess_prefix" - elif test "x$with_mpgen" != xno; then + elif test "$with_mpgen" != "no"; then MPGEN_PREFIX="$with_mpgen"; fi AC_SUBST(MPGEN_PREFIX) fi AC_MSG_CHECKING([whether to build bitcoind]) -AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) +AM_CONDITIONAL([BUILD_BITCOIND], [test $build_bitcoind = "yes"]) AC_MSG_RESULT($build_bitcoind) AC_MSG_CHECKING([whether to build bitcoin-cli]) -AM_CONDITIONAL([BUILD_BITCOIN_CLI], [test x$build_bitcoin_cli = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_CLI], [test $build_bitcoin_cli = "yes"]) AC_MSG_RESULT($build_bitcoin_cli) AC_MSG_CHECKING([whether to build bitcoin-tx]) -AM_CONDITIONAL([BUILD_BITCOIN_TX], [test x$build_bitcoin_tx = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_TX], [test $build_bitcoin_tx = "yes"]) AC_MSG_RESULT($build_bitcoin_tx) AC_MSG_CHECKING([whether to build bitcoin-wallet]) -AM_CONDITIONAL([BUILD_BITCOIN_WALLET], [test x$build_bitcoin_wallet = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_WALLET], [test $build_bitcoin_wallet = "yes"]) AC_MSG_RESULT($build_bitcoin_wallet) AC_MSG_CHECKING([whether to build bitcoin-util]) -AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test x$build_bitcoin_util = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test $build_bitcoin_util = "yes"]) AC_MSG_RESULT($build_bitcoin_util) +AC_MSG_CHECKING([whether to build experimental bitcoin-chainstate]) +if test "$build_bitcoin_chainstate" = "yes"; then + if test "$build_experimental_kernel_lib" = "no"; then + AC_MSG_ERROR([experimental bitcoin-chainstate cannot be built without the experimental bitcoinkernel library. Use --with-experimental-kernel-lib]); + fi +fi +AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"]) +AC_MSG_RESULT($build_bitcoin_chainstate) + AC_MSG_CHECKING([whether to build libraries]) -AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test x$build_bitcoin_libs = xyes]) -if test x$build_bitcoin_libs = xyes; then - AC_DEFINE(HAVE_CONSENSUS_LIB, 1, [Define this symbol if the consensus lib has been built]) +AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"]) + +if test "$build_bitcoin_libs" = "yes"; then + AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built]) AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) fi + +AM_CONDITIONAL([BUILD_BITCOIN_KERNEL_LIB], [test "$build_experimental_kernel_lib" != "no" && ( test "$build_experimental_kernel_lib" = "yes" || test "$build_bitcoin_chainstate" = "yes" )]) + AC_MSG_RESULT($build_bitcoin_libs) AC_LANG_POP -if test "x$use_ccache" != "xno"; then - AC_MSG_CHECKING(if ccache should be used) - if test x$CCACHE = x; then - if test "x$use_ccache" = "xyes"; then +if test "$use_ccache" != "no"; then + AC_MSG_CHECKING([if ccache should be used]) + if test "$CCACHE" = ""; then + if test "$use_ccache" = "yes"; then AC_MSG_ERROR([ccache not found.]); else use_ccache=no @@ -1615,73 +1737,73 @@ if test "x$use_ccache" != "xno"; then CXX="$ac_cv_path_CCACHE $CXX" fi AC_MSG_RESULT($use_ccache) - if test "x$use_ccache" = "xyes"; then - AX_CHECK_COMPILE_FLAG([-fdebug-prefix-map=A=B],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -fdebug-prefix-map=\$(abs_srcdir)=."],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-fmacro-prefix-map=A=B],[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -fmacro-prefix-map=\$(abs_srcdir)=."],,[[$CXXFLAG_WERROR]]) + if test "$use_ccache" = "yes"; then + AX_CHECK_COMPILE_FLAG([-fdebug-prefix-map=A=B], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -fdebug-prefix-map=\$(abs_top_srcdir)=."], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-fmacro-prefix-map=A=B], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -fmacro-prefix-map=\$(abs_top_srcdir)=."], [], [$CXXFLAG_WERROR]) fi fi dnl enable wallet AC_MSG_CHECKING([if wallet should be enabled]) -if test x$enable_wallet != xno; then - AC_MSG_RESULT(yes) +if test "$enable_wallet" != "no"; then + AC_MSG_RESULT([yes]) AC_DEFINE_UNQUOTED([ENABLE_WALLET],[1],[Define to 1 to enable wallet functions]) enable_wallet=yes else - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) fi dnl enable upnp support AC_MSG_CHECKING([whether to build with support for UPnP]) -if test x$have_miniupnpc = xno; then - if test x$use_upnp = xyes; then - AC_MSG_ERROR("UPnP requested but cannot be built. Use --without-miniupnpc.") +if test "$have_miniupnpc" = "no"; then + if test "$use_upnp" = "yes"; then + AC_MSG_ERROR([UPnP requested but cannot be built. Use --without-miniupnpc]) fi - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) use_upnp=no else - if test x$use_upnp != xno; then - AC_MSG_RESULT(yes) + if test "$use_upnp" != "no"; then + AC_MSG_RESULT([yes]) AC_MSG_CHECKING([whether to build with UPnP enabled by default]) use_upnp=yes upnp_setting=0 - if test x$use_upnp_default != xno; then + if test "$use_upnp_default" != "no"; then use_upnp_default=yes upnp_setting=1 fi - AC_MSG_RESULT($use_upnp_default) + AC_MSG_RESULT([$use_upnp_default]) AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state]) - if test x$TARGET_OS = xwindows; then - MINIUPNPC_CPPFLAGS="-DSTATICLIB -DMINIUPNP_STATICLIB" + if test "$TARGET_OS" = "windows"; then + MINIUPNPC_CPPFLAGS="$MINIUPNPC_CPPFLAGS -DSTATICLIB -DMINIUPNP_STATICLIB" fi else - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) fi fi dnl Enable NAT-PMP support. AC_MSG_CHECKING([whether to build with support for NAT-PMP]) -if test "x$have_natpmp" = xno; then - if test "x$use_natpmp" = xyes; then +if test "$have_natpmp" = "no"; then + if test "$use_natpmp" = "yes"; then AC_MSG_ERROR([NAT-PMP requested but cannot be built. Use --without-natpmp]) fi AC_MSG_RESULT([no]) use_natpmp=no else - if test "x$use_natpmp" != xno; then + if test "$use_natpmp" != "no"; then AC_MSG_RESULT([yes]) AC_MSG_CHECKING([whether to build with NAT-PMP enabled by default]) use_natpmp=yes natpmp_setting=0 - if test "x$use_natpmp_default" != xno; then + if test "$use_natpmp_default" != "no"; then use_natpmp_default=yes natpmp_setting=1 fi AC_MSG_RESULT($use_natpmp_default) AC_DEFINE_UNQUOTED([USE_NATPMP], [$natpmp_setting], [NAT-PMP support not compiled if undefined, otherwise value (0 or 1) determines default state]) - if test x$TARGET_OS = xwindows; then - NATPMP_CPPFLAGS="-DSTATICLIB -DNATPMP_STATICLIB" + if test "$TARGET_OS" = "windows"; then + NATPMP_CPPFLAGS="$NATPMP_CPPFLAGS -DSTATICLIB -DNATPMP_STATICLIB" fi else AC_MSG_RESULT([no]) @@ -1690,35 +1812,35 @@ fi dnl these are only used when qt is enabled BUILD_TEST_QT="" -if test x$bitcoin_enable_qt != xno; then +if test "$bitcoin_enable_qt" != "no"; then dnl enable dbus support AC_MSG_CHECKING([whether to build GUI with support for D-Bus]) - if test x$bitcoin_enable_qt_dbus != xno; then - AC_DEFINE([USE_DBUS],[1],[Define if dbus support should be compiled in]) + if test "$bitcoin_enable_qt_dbus" != "no"; then + AC_DEFINE([USE_DBUS], [1], [Define if dbus support should be compiled in]) fi - AC_MSG_RESULT($bitcoin_enable_qt_dbus) + AC_MSG_RESULT([$bitcoin_enable_qt_dbus]) dnl enable qr support AC_MSG_CHECKING([whether to build GUI with support for QR codes]) - if test x$have_qrencode = xno; then - if test x$use_qr = xyes; then + if test "$have_qrencode" = "no"; then + if test "$use_qr" = "yes"; then AC_MSG_ERROR([QR support requested but cannot be built. Use --without-qrencode]) fi use_qr=no else - if test x$use_qr != xno; then - AC_DEFINE([USE_QRCODE],[1],[Define if QR support should be compiled in]) + if test "$use_qr" != "no"; then + AC_DEFINE([USE_QRCODE], [1], [Define if QR support should be compiled in]) use_qr=yes fi fi AC_MSG_RESULT([$use_qr]) - if test x$XGETTEXT = x; then - AC_MSG_WARN("xgettext is required to update qt translations") + if test "$XGETTEXT" = ""; then + AC_MSG_WARN([xgettext is required to update qt translations]) fi AC_MSG_CHECKING([whether to build test_bitcoin-qt]) - if test x$use_gui_tests$bitcoin_enable_qt_test = xyesyes; then + if test "$use_gui_tests$bitcoin_enable_qt_test" = "yesyes"; then AC_MSG_RESULT([yes]) BUILD_TEST_QT="yes" else @@ -1726,11 +1848,11 @@ if test x$bitcoin_enable_qt != xno; then fi fi -AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"]) +AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) AC_MSG_CHECKING([whether to build test_bitcoin]) -if test x$use_tests = xyes; then - if test "x$enable_fuzz" = "xyes"; then +if test "$use_tests" = "yes"; then + if test "$enable_fuzz" = "yes"; then AC_MSG_RESULT([no, because fuzzing is enabled]) else AC_MSG_RESULT([yes]) @@ -1742,54 +1864,57 @@ else fi AC_MSG_CHECKING([whether to reduce exports]) -if test x$use_reduce_exports = xyes; then +if test "$use_reduce_exports" = "yes"; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) fi -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests = xnononononononono; then - AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-bench or --enable-tests]) +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nonononononononono"; then + AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-fuzz(-binary) --enable-bench or --enable-tests]) fi -AM_CONDITIONAL([TARGET_DARWIN], [test x$TARGET_OS = xdarwin]) -AM_CONDITIONAL([BUILD_DARWIN], [test x$BUILD_OS = xdarwin]) -AM_CONDITIONAL([TARGET_LINUX], [test x$TARGET_OS = xlinux]) -AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) -AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) -AM_CONDITIONAL([USE_SQLITE], [test "x$use_sqlite" = "xyes"]) -AM_CONDITIONAL([USE_BDB], [test "x$use_bdb" = "xyes"]) -AM_CONDITIONAL([ENABLE_TRACING],[test x$have_sdt = xyes]) -AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) -AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes]) -AM_CONDITIONAL([ENABLE_FUZZ_BINARY],[test x$enable_fuzz_binary = xyes]) -AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) -AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) -AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) -AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) -AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) -AM_CONDITIONAL([USE_LIBEVENT],[test x$use_libevent = xyes]) -AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) -AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) -AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes]) -AM_CONDITIONAL([ENABLE_SSE41],[test x$enable_sse41 = xyes]) -AM_CONDITIONAL([ENABLE_AVX2],[test x$enable_avx2 = xyes]) -AM_CONDITIONAL([ENABLE_SHANI],[test x$enable_shani = xyes]) -AM_CONDITIONAL([ENABLE_ARM_CRC],[test x$enable_arm_crc = xyes]) -AM_CONDITIONAL([USE_ASM],[test x$use_asm = xyes]) -AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes]) -AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes]) -AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes]) - -AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) -AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) -AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build]) -AC_DEFINE(CLIENT_VERSION_IS_RELEASE, _CLIENT_VERSION_IS_RELEASE, [Version is release]) -AC_DEFINE(COPYRIGHT_YEAR, _COPYRIGHT_YEAR, [Copyright year]) -AC_DEFINE(COPYRIGHT_HOLDERS, "_COPYRIGHT_HOLDERS", [Copyright holder(s) before %s replacement]) -AC_DEFINE(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION", [Replacement for %s in copyright holders string]) +AM_CONDITIONAL([TARGET_DARWIN], [test "$TARGET_OS" = "darwin"]) +AM_CONDITIONAL([BUILD_DARWIN], [test "$BUILD_OS" = "darwin"]) +AM_CONDITIONAL([TARGET_LINUX], [test "$TARGET_OS" = "linux"]) +AM_CONDITIONAL([TARGET_WINDOWS], [test "$TARGET_OS" = "windows"]) +AM_CONDITIONAL([ENABLE_WALLET], [test "$enable_wallet" = "yes"]) +AM_CONDITIONAL([USE_SQLITE], [test "$use_sqlite" = "yes"]) +AM_CONDITIONAL([USE_BDB], [test "$use_bdb" = "yes"]) +AM_CONDITIONAL([ENABLE_TESTS], [test "$BUILD_TEST" = "yes"]) +AM_CONDITIONAL([ENABLE_FUZZ], [test "$enable_fuzz" = "yes"]) +AM_CONDITIONAL([ENABLE_FUZZ_BINARY], [test "$enable_fuzz_binary" = "yes"]) +AM_CONDITIONAL([ENABLE_QT], [test "$bitcoin_enable_qt" = "yes"]) +AM_CONDITIONAL([ENABLE_QT_TESTS], [test "$BUILD_TEST_QT" = "yes"]) +AM_CONDITIONAL([ENABLE_BENCH], [test "$use_bench" = "yes"]) +AM_CONDITIONAL([USE_QRCODE], [test "$use_qr" = "yes"]) +AM_CONDITIONAL([USE_LCOV], [test "$use_lcov" = "yes"]) +AM_CONDITIONAL([USE_LIBEVENT], [test "$use_libevent" = "yes"]) +AM_CONDITIONAL([HARDEN], [test "$use_hardening" = "yes"]) +AM_CONDITIONAL([ENABLE_SSE42], [test "$enable_sse42" = "yes"]) +AM_CONDITIONAL([ENABLE_SSE41], [test "$enable_sse41" = "yes"]) +AM_CONDITIONAL([ENABLE_AVX2], [test "$enable_avx2" = "yes"]) +AM_CONDITIONAL([ENABLE_X86_SHANI], [test "$enable_x86_shani" = "yes"]) +AM_CONDITIONAL([ENABLE_ARM_CRC], [test "$enable_arm_crc" = "yes"]) +AM_CONDITIONAL([ENABLE_ARM_SHANI], [test "$enable_arm_shani" = "yes"]) +AM_CONDITIONAL([USE_ASM], [test "$use_asm" = "yes"]) +AM_CONDITIONAL([WORDS_BIGENDIAN], [test "$ac_cv_c_bigendian" = "yes"]) +AM_CONDITIONAL([USE_NATPMP], [test "$use_natpmp" = "yes"]) +AM_CONDITIONAL([USE_UPNP], [test "$use_upnp" = "yes"]) + +dnl for minisketch +AM_CONDITIONAL([ENABLE_CLMUL], [test "$enable_clmul" = "yes"]) +AM_CONDITIONAL([HAVE_CLZ], [test "$have_clzl$have_clzll" = "yesyes"]) + +AC_DEFINE([CLIENT_VERSION_MAJOR], [_CLIENT_VERSION_MAJOR], [Major version]) +AC_DEFINE([CLIENT_VERSION_MINOR], [_CLIENT_VERSION_MINOR], [Minor version]) +AC_DEFINE([CLIENT_VERSION_BUILD], [_CLIENT_VERSION_BUILD], [Version Build]) +AC_DEFINE([CLIENT_VERSION_IS_RELEASE], [_CLIENT_VERSION_IS_RELEASE], [Version is release]) +AC_DEFINE([COPYRIGHT_YEAR], [_COPYRIGHT_YEAR], [Copyright year]) +AC_DEFINE([COPYRIGHT_HOLDERS], ["_COPYRIGHT_HOLDERS"], [Copyright holder(s) before %s replacement]) +AC_DEFINE([COPYRIGHT_HOLDERS_SUBSTITUTION], ["_COPYRIGHT_HOLDERS_SUBSTITUTION"], [Replacement for %s in copyright holders string]) define(_COPYRIGHT_HOLDERS_FINAL, [patsubst(_COPYRIGHT_HOLDERS, [%s], [_COPYRIGHT_HOLDERS_SUBSTITUTION])]) -AC_DEFINE(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL", [Copyright holder(s)]) +AC_DEFINE([COPYRIGHT_HOLDERS_FINAL], ["_COPYRIGHT_HOLDERS_FINAL"], [Copyright holder(s)]) AC_SUBST(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR) AC_SUBST(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR) AC_SUBST(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD) @@ -1800,51 +1925,53 @@ AC_SUBST(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION") AC_SUBST(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL") AC_SUBST(BITCOIN_DAEMON_NAME) AC_SUBST(BITCOIN_GUI_NAME) +AC_SUBST(BITCOIN_TEST_NAME) AC_SUBST(BITCOIN_CLI_NAME) AC_SUBST(BITCOIN_TX_NAME) AC_SUBST(BITCOIN_UTIL_NAME) +AC_SUBST(BITCOIN_CHAINSTATE_NAME) AC_SUBST(BITCOIN_WALLET_TOOL_NAME) AC_SUBST(BITCOIN_MP_NODE_NAME) AC_SUBST(BITCOIN_MP_GUI_NAME) AC_SUBST(RELDFLAGS) +AC_SUBST(CORE_LDFLAGS) +AC_SUBST(CORE_CPPFLAGS) +AC_SUBST(CORE_CXXFLAGS) AC_SUBST(DEBUG_CPPFLAGS) AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) AC_SUBST(DEBUG_CXXFLAGS) -AC_SUBST(COMPAT_LDFLAGS) AC_SUBST(ERROR_CXXFLAGS) AC_SUBST(GPROF_CXXFLAGS) AC_SUBST(GPROF_LDFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) +AC_SUBST(LTO_CXXFLAGS) +AC_SUBST(LTO_LDFLAGS) AC_SUBST(PIC_FLAGS) AC_SUBST(PIE_FLAGS) AC_SUBST(SANITIZER_CXXFLAGS) AC_SUBST(SANITIZER_LDFLAGS) AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(SSE41_CXXFLAGS) +AC_SUBST(CLMUL_CXXFLAGS) AC_SUBST(AVX2_CXXFLAGS) -AC_SUBST(SHANI_CXXFLAGS) +AC_SUBST(X86_SHANI_CXXFLAGS) AC_SUBST(ARM_CRC_CXXFLAGS) +AC_SUBST(ARM_SHANI_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_SQLITE) AC_SUBST(USE_BDB) AC_SUBST(ENABLE_EXTERNAL_SIGNER) AC_SUBST(USE_UPNP) AC_SUBST(USE_QRCODE) -AC_SUBST(BOOST_LIBS) -AC_SUBST(SQLITE_LIBS) AC_SUBST(TESTDEFS) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) AC_SUBST(NATPMP_CPPFLAGS) AC_SUBST(NATPMP_LIBS) -AC_SUBST(EVENT_LIBS) -AC_SUBST(EVENT_PTHREADS_LIBS) -AC_SUBST(ZMQ_LIBS) -AC_SUBST(QR_LIBS) AC_SUBST(HAVE_GMTIME_R) AC_SUBST(HAVE_FDATASYNC) AC_SUBST(HAVE_FULLFSYNC) @@ -1852,20 +1979,23 @@ AC_SUBST(HAVE_O_CLOEXEC) AC_SUBST(HAVE_BUILTIN_PREFETCH) AC_SUBST(HAVE_MM_PREFETCH) AC_SUBST(HAVE_STRONG_GETAUXVAL) -AC_SUBST(HAVE_WEAK_GETAUXVAL) AC_SUBST(ANDROID_ARCH) +AC_SUBST(HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])]) -AC_CONFIG_LINKS([contrib/devtools/security-check.py:contrib/devtools/security-check.py]) -AC_CONFIG_LINKS([contrib/devtools/symbol-check.py:contrib/devtools/symbol-check.py]) -AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py]) -AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py]) +AC_CONFIG_LINKS([contrib/devtools/iwyu/bitcoin.core.imp:contrib/devtools/iwyu/bitcoin.core.imp]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) +AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff]) +AC_CONFIG_LINKS([src/.bear-tidy-config:src/.bear-tidy-config]) +AC_CONFIG_LINKS([src/.clang-tidy:src/.clang-tidy]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) -AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py]) +AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py]) AC_CONFIG_LINKS([test/util/rpcauth-test.py:test/util/rpcauth-test.py]) +AC_CONFIG_LINKS([src/qt/Makefile:src/qt/Makefile]) +AC_CONFIG_LINKS([src/qt/test/Makefile:src/qt/test/Makefile]) +AC_CONFIG_LINKS([src/test/Makefile:src/test/Makefile]) dnl boost's m4 checks do something really nasty: they export these vars. As a dnl result, they leak into secp256k1's configure and crazy things happen. @@ -1882,19 +2012,7 @@ LIBS_TEMP="$LIBS" unset LIBS LIBS="$LIBS_TEMP" -PKGCONFIG_PATH_TEMP="$PKG_CONFIG_PATH" -unset PKG_CONFIG_PATH -PKG_CONFIG_PATH="$PKGCONFIG_PATH_TEMP" - -PKGCONFIG_LIBDIR_TEMP="$PKG_CONFIG_LIBDIR" -unset PKG_CONFIG_LIBDIR -PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP" - -if test x$need_bundled_univalue = xyes; then - AC_CONFIG_SUBDIRS([src/univalue]) -fi - -ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental" +ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT @@ -1911,41 +2029,44 @@ echo echo "Options used to compile and link:" echo " external signer = $use_external_signer" echo " multiprocess = $build_multiprocess" +echo " with experimental syscall sandbox support = $use_syscall_sandbox" echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" -if test "x$enable_wallet" != "xno"; then +if test "$enable_wallet" != "no"; then echo " with sqlite = $use_sqlite" echo " with bdb = $use_bdb" fi echo " with gui / qt = $bitcoin_enable_qt" -if test x$bitcoin_enable_qt != xno; then +if test $bitcoin_enable_qt != "no"; then echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" -if test x$enable_fuzz == xno; then +if test $enable_fuzz = "no"; then echo " with test = $use_tests" else echo " with test = not building test_bitcoin because fuzzing is enabled" - echo " with fuzz = $enable_fuzz" fi +echo " with fuzz binary = $enable_fuzz_binary" echo " with bench = $use_bench" echo " with upnp = $use_upnp" echo " with natpmp = $use_natpmp" echo " use asm = $use_asm" -echo " ebpf tracing = $have_sdt" +echo " USDT tracing = $use_usdt" echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" +echo " LTO = $enable_lto" echo -echo " target os = $TARGET_OS" +echo " target os = $host_os" echo " build os = $build_os" echo echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" -echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" +echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CORE_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $CORE_LDFLAGS $LDFLAGS" +echo " AR = $AR" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/README.md b/contrib/README.md index a2612ab95845..0252d668d9af 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -26,18 +26,12 @@ The [Debian](/contrib/debian) subfolder contains the copyright file. All other packaging related files can be found in the [bitcoin-core/packaging](https://github.com/bitcoin-core/packaging) repository. -### [Gitian-descriptors](/contrib/gitian-descriptors) ### -Files used during the gitian build process. For more information about gitian, see the [the Bitcoin Core documentation repository](https://github.com/bitcoin-core/docs). - ### [Builder keys](/contrib/builder-keys) PGP keys used for signing Bitcoin Core [release](/doc/release-process.md) results. ### [MacDeploy](/contrib/macdeploy) ### Scripts and notes for Mac builds. -### [Gitian-build](/contrib/gitian-build.py) ### -Script for running full Gitian builds. - Test and Verify Tools --------------------- @@ -46,3 +40,9 @@ Utilities to generate test vectors for the data-driven Bitcoin tests. ### [Verify Binaries](/contrib/verifybinaries) ### This script attempts to download and verify the signature file SHA256SUMS.asc from bitcoin.org. + +Command Line Tools +--------------------- + +### [Completions](/contrib/completions) ### +Shell completions for bash and fish. diff --git a/contrib/builder-keys/README.md b/contrib/builder-keys/README.md index a7c1d5ae0a75..a6179d6012cd 100644 --- a/contrib/builder-keys/README.md +++ b/contrib/builder-keys/README.md @@ -19,8 +19,14 @@ gpg --refresh-keys To fetch keys of builders and active developers, feed the list of fingerprints of the primary keys into gpg: +On \*NIX: ```sh -while read fingerprint keyholder_name; do gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys ${fingerprint}; done < ./keys.txt +while read fingerprint keyholder_name; do gpg --keyserver hkps://keys.openpgp.org --recv-keys ${fingerprint}; done < ./keys.txt +``` + +On Windows (requires Gpg4win >= 4.0.0): +``` +FOR /F "tokens=1" %i IN (keys.txt) DO gpg --keyserver hkps://keys.openpgp.org --recv-keys %i ``` Add your key to the list if you provided Guix attestations for two major or diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index db28cd07a0b0..b2b5b031f8a6 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -1,3 +1,4 @@ +982A193E3CE0EED535E09023188CBB2648416AD5 0xB10C (0xb10c) 9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C Aaron Clauson (sipsorcery) 617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa (akx20000) E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) @@ -5,6 +6,7 @@ E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) 590B7292695AFFA5B672CBB2E13FC145CD3F4304 Antoine Poinsot (darosior) 0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 Ben Carman (benthecarman) 912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak (btcdrak) +04017A2A6D9A0CCDC81D8EC296AB007F1A7ED999 Carl Dong (dongcarl) C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) 18AE2F798E0D239755DA4FD24B79F986CBDF8736 Chun Kuan Le (ken2812221) 101598DC823C1B5F9A6624ABA5E0907A0380E6C3 CoinForensics (CoinForensics) @@ -12,13 +14,15 @@ F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur) C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields) BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) 6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark) +948444FCE03B05BA5AB0591EC37B1C1D44C786EE Duncan Dean (dunxen) 1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba) 9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) 6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls) 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) +6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C Gloria Zhao (glozow) D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) -A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) +2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob) D3F22A3A4C366C2DCB66D3722DA9C5A7FA81EA35 Jarol Rodriguez (jarolrod) 7480909378D544EA6B6DCEB7535B12980BB8A4D3 Jeffri H Frontz (jhfrontz) D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) @@ -26,6 +30,7 @@ D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli (jonasschnelli) 4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon (jtimon) C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) +70A1D47DD44F59DF8B22244333E472FE870C7E5D Kristaps Kaupe (kristapsk) 30DE693AE0DE9E37B3E7EB6BBFF0F67810C1EED1 Lisa Neigut (niftynei) E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr (luke-jr) B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke (marco) @@ -47,5 +52,6 @@ ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost (sjors) 9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) 6DEEF79B050C4072509B743F8C275BC595448867 Tomas Kanocz (KanoczTomas) AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami (wtogami) +74E2DEF5D77260B98BC19438099BAD163C70FBFA Will Clark (will8clark) 79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko (willyko) 71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan (laanwj) diff --git a/contrib/bitcoin-cli.bash-completion b/contrib/completions/bash/bitcoin-cli.bash-completion similarity index 100% rename from contrib/bitcoin-cli.bash-completion rename to contrib/completions/bash/bitcoin-cli.bash-completion diff --git a/contrib/bitcoin-tx.bash-completion b/contrib/completions/bash/bitcoin-tx.bash-completion similarity index 100% rename from contrib/bitcoin-tx.bash-completion rename to contrib/completions/bash/bitcoin-tx.bash-completion diff --git a/contrib/bitcoind.bash-completion b/contrib/completions/bash/bitcoind.bash-completion similarity index 100% rename from contrib/bitcoind.bash-completion rename to contrib/completions/bash/bitcoind.bash-completion diff --git a/contrib/completions/fish/bitcoin-cli.fish b/contrib/completions/fish/bitcoin-cli.fish new file mode 100644 index 000000000000..2f034c475c1d --- /dev/null +++ b/contrib/completions/fish/bitcoin-cli.fish @@ -0,0 +1,99 @@ +# Disable files from being included in completions by default +complete --command bitcoin-cli --no-files + +function __fish_bitcoin_cli_get_commands_helper + set --local cmd (commandline -oc) + + # Don't return commands if '-help or -?' in commandline + if string match --quiet --regex -- '^-help$|^-\?$' $cmd + return + end + + # Strip help cmd from token to avoid duplication errors + set --local cmd (string match --invert --regex -- '^help$' $cmd) + # Strip -stdin* options to avoid waiting for input while we fetch completions + # TODO: this appears to be broken when run as tab completion (requires ctrl+c to exit) + set --local cmd (string match --invert --regex -- '^-stdin.*$' $cmd) + + # Match, format and return commands + for command in ($cmd help 2>&1 | string match --invert -r '^\=\=.*' | string match --invert -r '^\\s*$') + echo $command + end +end + +function __fish_bitcoin_cli_get_commands + argparse 'nohelp' 'commandsonly' -- $argv + set --local commands + + # Exclude description, exclude help + if set -q _flag_nohelp; and set -q _flag_commandsonly + set --append commands (__fish_bitcoin_cli_get_commands_helper | string replace -r ' .*$' '' | string match --invert -r 'help') + # Include description, exclude help + else if set -q _flag_nohelp + set --append commands (__fish_bitcoin_cli_get_commands_helper | string replace ' ' \t | string match --invert -r 'help') + # Exclude description, include help + else if set -q _flag_commandsonly + set --append commands (__fish_bitcoin_cli_get_commands_helper | string replace -r ' .*$' '') + # Include description, include help + else + set --append commands (__fish_bitcoin_cli_get_commands_helper | string replace ' ' \t) + end + + if string match -q -r '^.*error.*$' $commands[1] + # RPC offline or RPC wallet not loaded + return + else + for command in $commands + echo $command + end + end +end + + +function __fish_bitcoin_cli_get_options + argparse 'nofiles' -- $argv + set --local cmd (commandline -oc) + # Don't return options if '-help or -?' in commandline + if string match --quiet --regex -- '^-help$|-\?$' $cmd + return + end + set --local options + + if set -q _flag_nofiles + set --append options ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match --invert -r '^.*=$') + else + set --append options ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match -r '^.*=$') + end + + for option in $options + echo $option + end +end + +# Add options with file completion +# Don't offer after a command is given +complete \ + --command bitcoin-cli \ + --no-files \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_cli_get_commands --commandsonly)" \ + --arguments "(__fish_bitcoin_cli_get_options)" +# Enable file completions only if the commandline now contains a `*.=` style option +complete --command bitcoin-cli \ + --condition 'string match --regex -- ".*=" (commandline -pt)' \ + --force-files + +# Add options without file completion +# Don't offer after a command is given +complete \ + --command bitcoin-cli \ + --no-files \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_cli_get_commands --commandsonly)" \ + --arguments "(__fish_bitcoin_cli_get_options --nofiles)" + +# Add commands +# Permit command completions after `bitcoin-cli help` but not after other commands +complete \ + --command bitcoin-cli \ + --no-files \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_cli_get_commands --commandsonly --nohelp)" \ + --arguments "(__fish_bitcoin_cli_get_commands)" diff --git a/contrib/completions/fish/bitcoin-qt.fish b/contrib/completions/fish/bitcoin-qt.fish new file mode 100644 index 000000000000..15a355ae88f5 --- /dev/null +++ b/contrib/completions/fish/bitcoin-qt.fish @@ -0,0 +1,35 @@ +# Disable files from being included in completions by default +complete --command bitcoin-qt --no-files + +# Extract options +function __fish_bitcoinqt_get_options + argparse 'nofiles' -- $argv + set --local cmd (commandline -opc)[1] + set --local options + + if set -q _flag_nofiles + set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match --invert -r '^.*=$') + else + set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match -r '^.*=$') + end + + for option in $options + echo $option + end +end + + +# Add options with file completion +complete \ + --command bitcoin-qt \ + --arguments "(__fish_bitcoinqt_get_options)" +# Enable file completions only if the commandline now contains a `*.=` style option +complete -c bitcoin-qt \ + --condition 'string match --regex -- ".*=" (commandline -pt)' \ + --force-files + +# Add options without file completion +complete \ + --command bitcoin-qt \ + --arguments "(__fish_bitcoinqt_get_options --nofiles)" + diff --git a/contrib/completions/fish/bitcoin-tx.fish b/contrib/completions/fish/bitcoin-tx.fish new file mode 100644 index 000000000000..0ff262b948e4 --- /dev/null +++ b/contrib/completions/fish/bitcoin-tx.fish @@ -0,0 +1,65 @@ +# Disable files from being included in completions by default +complete --command bitcoin-tx --no-files + +# Modified version of __fish_seen_subcommand_from +# Uses regex to detect cmd= syntax +function __fish_bitcoin_seen_cmd + set -l cmd (commandline -oc) + set -e cmd[1] + for i in $cmd + for j in $argv + if string match --quiet --regex -- "^$j.*" $i + return 0 + end + end + end + return 1 +end + +# Extract options +function __fish_bitcoin_tx_get_options + set --local cmd (commandline -oc)[1] + if string match --quiet --regex -- '^-help$|-\?$' $cmd + return + end + + for option in ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=') + echo $option + end +end + +# Extract commands +function __fish_bitcoin_tx_get_commands + argparse 'commandsonly' -- $argv + set --local cmd (commandline -oc)[1] + set --local commands + + if set -q _flag_commandsonly + set --append commands ($cmd -help | sed -e '1,/Commands:/d' -e 's/=/=\t/' -e 's/(=/=/' -e '/^ [a-z]/ p' -e d | string replace -r '\ \ ' '' | string replace -r '=.*' '') + else + set --append commands ($cmd -help | sed -e '1,/Commands:/d' -e 's/=/=\t/' -e 's/(=/=/' -e '/^ [a-z]/ p' -e d | string replace -r '\ \ ' '') + end + + for command in $commands + echo $command + end +end + +# Add options +complete \ + --command bitcoin-tx \ + --condition "not __fish_bitcoin_seen_cmd (__fish_bitcoin_tx_get_commands --commandsonly)" \ + --arguments "(__fish_bitcoin_tx_get_options)" \ + --no-files + +# Add commands +complete \ + --command bitcoin-tx \ + --arguments "(__fish_bitcoin_tx_get_commands)" \ + --no-files + +# Add file completions for load and set commands +complete \ + --command bitcoin-tx \ + --condition 'string match --regex -- "(load|set)=" (commandline -pt)' \ + --force-files diff --git a/contrib/completions/fish/bitcoin-util.fish b/contrib/completions/fish/bitcoin-util.fish new file mode 100644 index 000000000000..0650bf2cb6d3 --- /dev/null +++ b/contrib/completions/fish/bitcoin-util.fish @@ -0,0 +1,38 @@ +# Disable files from being included in completions by default +complete --command bitcoin-util --no-files + +# Extract options +function __fish_bitcoin_util_get_options + set --local cmd (commandline -opc)[1] + set --local options + + set --append options ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=') + + for option in $options + echo $option + end +end + +# Extract commands +function __fish_bitcoin_util_get_commands + set --local cmd (commandline -opc)[1] + set --local commands + + set --append commands ($cmd -help | sed -e '1,/Commands:/d' -e 's/=/=\t/' -e 's/(=/=/' -e '/^ [a-z]/ p' -e d | string replace -r '\ \ ' '') + for command in $commands + echo $command + end +end + +# Add options +complete \ + --command bitcoin-util \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_util_get_commands)" \ + --arguments "(__fish_bitcoin_util_get_options)" + +# Add commands +complete \ + --command bitcoin-util \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_util_get_commands)" \ + --arguments "(__fish_bitcoin_util_get_commands)" + diff --git a/contrib/completions/fish/bitcoin-wallet.fish b/contrib/completions/fish/bitcoin-wallet.fish new file mode 100644 index 000000000000..82d8277c9b4f --- /dev/null +++ b/contrib/completions/fish/bitcoin-wallet.fish @@ -0,0 +1,35 @@ +# Disable files from being included in completions by default +complete --command bitcoin-wallet --no-files + +# Extract options +function __fish_bitcoin_wallet_get_options + set --local cmd (commandline -opc)[1] + for option in ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=') + echo $option + end +end + +# Extract commands +function __fish_bitcoin_wallet_get_commands + set --local cmd (commandline -opc)[1] + for command in ($cmd -help | sed -e '1,/Commands:/d' -e 's/=/=\t/' -e 's/(=/=/' -e '/^ [a-z]/ p' -e d | string replace -r '\ \ ' '') + echo $command + end +end + +# Add options +complete \ + --command bitcoin-wallet \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_wallet_get_commands)" \ + --arguments "(__fish_bitcoin_wallet_get_options)" + +# Add commands +complete \ + --command bitcoin-wallet \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_wallet_get_commands)" \ + --arguments "(__fish_bitcoin_wallet_get_commands)" + +# Add file completions for load and set commands +complete --command bitcoin-wallet \ + --condition "string match -r -- '(dumpfile|datadir)*=' (commandline -pt)" \ + --force-files diff --git a/contrib/completions/fish/bitcoind.fish b/contrib/completions/fish/bitcoind.fish new file mode 100644 index 000000000000..fa245ae17f47 --- /dev/null +++ b/contrib/completions/fish/bitcoind.fish @@ -0,0 +1,35 @@ +# Disable files from being included in completions by default +complete --command bitcoind --no-files + +# Extract options +function __fish_bitcoind_get_options + argparse 'nofiles' -- $argv + set --local cmd (commandline -opc)[1] + set --local options + + if set -q _flag_nofiles + set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match --invert -r '^.*=$') + else + set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match -r '^.*=$') + end + + for option in $options + echo $option + end +end + + +# Add options with file completion +complete \ + --command bitcoind \ + --arguments "(__fish_bitcoind_get_options)" +# Enable file completions only if the commandline now contains a `*.=` style option +complete --command bitcoind \ + --condition 'string match --regex -- ".*=" (commandline -pt)' \ + --force-files + +# Add options without file completion +complete \ + --command bitcoind \ + --arguments "(__fish_bitcoind_get_options --nofiles)" + diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 6d23f600c39b..95a281ce054e 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,7 +5,7 @@ Upstream-Contact: Satoshi Nakamoto Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2021, Bitcoin Core Developers +Copyright: 2009-2022, Bitcoin Core Developers License: Expat Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, as well as the numerous contributors to the project. diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 1fa850af1a7c..54b1a8558818 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -76,7 +76,7 @@ year rather than two hyphenated years. If the file already has a copyright for `The Bitcoin Core developers`, the script will exit. -gen-manpages.sh +gen-manpages.py =============== A small script to automatically create manpages in ../../doc/man by running the release binaries with the -help option. @@ -87,7 +87,22 @@ repostitory. To use this tool with out-of-tree builds set `BUILDDIR`. For example: ```bash -BUILDDIR=$PWD/build contrib/devtools/gen-manpages.sh +BUILDDIR=$PWD/build contrib/devtools/gen-manpages.py +``` + +gen-bitcoin-conf.sh +=================== + +Generates a bitcoin.conf file in `share/examples/` by parsing the output from `bitcoind --help`. This script is run during the +release process to include a bitcoin.conf with the release binaries and can also be run by users to generate a file locally. +When generating a file as part of the release process, make sure to commit the changes after running the script. + +With in-tree builds this tool can be run from any directory within the +repository. To use this tool with out-of-tree builds set `BUILDDIR`. For +example: + +```bash +BUILDDIR=$PWD/build contrib/devtools/gen-bitcoin-conf.sh ``` security-check.py and test-security-check.py @@ -98,7 +113,7 @@ Perform basic security checks on a series of executables. symbol-check.py =============== -A script to check that the executables produced by gitian only contain +A script to check that release executables only contain certain symbols and are only linked against allowed libraries. For Linux this means checking for allowed gcc, glibc and libstdc++ version symbols. @@ -106,9 +121,9 @@ This makes sure they are still compatible with the minimum supported distributio For macOS and Windows we check that the executables are only linked against libraries we allow. -Example usage after a gitian build: +Example usage: - find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py + find ../path/to/executables -type f -executable | xargs python3 contrib/devtools/symbol-check.py If no errors occur the return value will be 0 and the output will be empty. diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 9a555c70bb49..680de1f1b3e4 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -33,8 +33,8 @@ # git subtrees "src/crypto/ctaes/", "src/leveldb/", + "src/minisketch", "src/secp256k1/", - "src/univalue/", "src/crc32c/", ] @@ -319,15 +319,13 @@ def get_most_recent_git_change_year(filename): ################################################################################ def read_file_lines(filename): - f = open(filename, 'r', encoding="utf8") - file_lines = f.readlines() - f.close() + with open(filename, 'r', encoding="utf8") as f: + file_lines = f.readlines() return file_lines def write_file_lines(filename, file_lines): - f = open(filename, 'w', encoding="utf8") - f.write(''.join(file_lines)) - f.close() + with open(filename, 'w', encoding="utf8") as f: + f.write(''.join(file_lines)) ################################################################################ # update header years execution @@ -370,7 +368,7 @@ def create_updated_copyright_line(line, last_git_change_year): space_split = after_copyright.split(' ') year_range = space_split[0] start_year, end_year = parse_year_range(year_range) - if end_year == last_git_change_year: + if end_year >= last_git_change_year: return line return (before_copyright + copyright_splitter + year_range_to_str(start_year, last_git_change_year) + ' ' + diff --git a/contrib/devtools/gen-bitcoin-conf.sh b/contrib/devtools/gen-bitcoin-conf.sh new file mode 100755 index 000000000000..2ebbd4202230 --- /dev/null +++ b/contrib/devtools/gen-bitcoin-conf.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} +BUILDDIR=${BUILDDIR:-$TOPDIR} +BINDIR=${BINDIR:-$BUILDDIR/src} +BITCOIND=${BITCOIND:-$BINDIR/bitcoind} +SHARE_EXAMPLES_DIR=${SHARE_EXAMPLES_DIR:-$TOPDIR/share/examples} +EXAMPLE_CONF_FILE=${EXAMPLE_CONF_FILE:-$SHARE_EXAMPLES_DIR/bitcoin.conf} + +[ ! -x "$BITCOIND" ] && echo "$BITCOIND not found or not executable." && exit 1 + +DIRTY="" +VERSION_OUTPUT=$($BITCOIND --version) +if [[ $VERSION_OUTPUT == *"dirty"* ]]; then + DIRTY="${DIRTY}${BITCOIND}\n" +fi + +if [ -n "$DIRTY" ] +then + echo -e "WARNING: $BITCOIND was built from a dirty tree.\n" + echo -e "To safely generate a bitcoin.conf file, please commit your changes to $BITCOIND, rebuild, then run this script again.\n" +fi + +echo 'Generating example bitcoin.conf file in share/examples/' + +# create the directory, if it doesn't exist +mkdir -p "${SHARE_EXAMPLES_DIR}" + +# create the header text +cat > "${EXAMPLE_CONF_FILE}" << 'EOF' +## +## bitcoin.conf configuration file. +## Generated by contrib/devtools/gen-bitcoin-conf.sh. +## +## Lines beginning with # are comments. +## All possible configuration options are provided. To use, copy this file +## to your data directory (default or specified by -datadir), uncomment +## options you would like to change, and save the file. +## + + +### Options +EOF + +# parse the output from bitcoind --help +# adding newlines is a bit funky to ensure portability for BSD +# see here for more details: https://stackoverflow.com/a/24575385 +${BITCOIND} --help \ + | sed '1,/Print this help message and exit/d' \ + | sed -E 's/^[[:space:]]{2}\-/#/' \ + | sed -E 's/^[[:space:]]{7}/# /' \ + | sed -E '/[=[:space:]]/!s/#.*$/&=1/' \ + | awk '/^#[a-z]/{x=$0;next}{if (NF==0) print x"\n",x="";else print}' \ + | sed 's,\(^[[:upper:]].*\)\:$,\ +### \1,' \ + | sed 's/[[:space:]]*$//' >> "${EXAMPLE_CONF_FILE}" + +# create the footer text +cat >> "${EXAMPLE_CONF_FILE}" << 'EOF' + +# [Sections] +# Most options will apply to all networks. To confine an option to a specific +# network, add it under the relevant section below. +# +# Note: If not specified under a network section, the options addnode, connect, +# port, bind, rpcport, rpcbind, and wallet will only apply to mainnet. + +# Options for mainnet +[main] + +# Options for testnet +[test] + +# Options for signet +[signet] + +# Options for regtest +[regtest] +EOF diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py new file mode 100755 index 000000000000..38cdb3acb8d1 --- /dev/null +++ b/contrib/devtools/gen-manpages.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +import os +import subprocess +import sys +import tempfile + +BINARIES = [ +'src/bitcoind', +'src/bitcoin-cli', +'src/bitcoin-tx', +'src/bitcoin-wallet', +'src/bitcoin-util', +'src/qt/bitcoin-qt', +] + +# Paths to external utilities. +git = os.getenv('GIT', 'git') +help2man = os.getenv('HELP2MAN', 'help2man') + +# If not otherwise specified, get top directory from git. +topdir = os.getenv('TOPDIR') +if not topdir: + r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, universal_newlines=True) + topdir = r.stdout.rstrip() + +# Get input and output directories. +builddir = os.getenv('BUILDDIR', topdir) +mandir = os.getenv('MANDIR', os.path.join(topdir, 'doc/man')) + +# Verify that all the required binaries are usable, and extract copyright +# message in a first pass. +versions = [] +for relpath in BINARIES: + abspath = os.path.join(builddir, relpath) + try: + r = subprocess.run([abspath, "--version"], stdout=subprocess.PIPE, check=True, universal_newlines=True) + except IOError: + print(f'{abspath} not found or not an executable', file=sys.stderr) + sys.exit(1) + # take first line (which must contain version) + verstr = r.stdout.splitlines()[0] + # last word of line is the actual version e.g. v22.99.0-5c6b3d5b3508 + verstr = verstr.split()[-1] + assert verstr.startswith('v') + # remaining lines are copyright + copyright = r.stdout.split('\n')[1:] + assert copyright[0].startswith('Copyright (C)') + + versions.append((abspath, verstr, copyright)) + +if any(verstr.endswith('-dirty') for (_, verstr, _) in versions): + print("WARNING: Binaries were built from a dirty tree.") + print('man pages generated from dirty binaries should NOT be committed.') + print('To properly generate man pages, please commit your changes (or discard them), rebuild, then run this script again.') + print() + +with tempfile.NamedTemporaryFile('w', suffix='.h2m') as footer: + # Create copyright footer, and write it to a temporary include file. + # Copyright is the same for all binaries, so just use the first. + footer.write('[COPYRIGHT]\n') + footer.write('\n'.join(versions[0][2]).strip()) + footer.flush() + + # Call the binaries through help2man to produce a manual page for each of them. + for (abspath, verstr, _) in versions: + outname = os.path.join(mandir, os.path.basename(abspath) + '.1') + print(f'Generating {outname}…') + subprocess.run([help2man, '-N', '--version-string=' + verstr, '--include=' + footer.name, '-o', outname, abspath], check=True) diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh deleted file mode 100755 index b7bf76ce77e0..000000000000 --- a/contrib/devtools/gen-manpages.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} -BUILDDIR=${BUILDDIR:-$TOPDIR} - -BINDIR=${BINDIR:-$BUILDDIR/src} -MANDIR=${MANDIR:-$TOPDIR/doc/man} - -BITCOIND=${BITCOIND:-$BINDIR/bitcoind} -BITCOINCLI=${BITCOINCLI:-$BINDIR/bitcoin-cli} -BITCOINTX=${BITCOINTX:-$BINDIR/bitcoin-tx} -WALLET_TOOL=${WALLET_TOOL:-$BINDIR/bitcoin-wallet} -BITCOINUTIL=${BITCOINQT:-$BINDIR/bitcoin-util} -BITCOINQT=${BITCOINQT:-$BINDIR/qt/bitcoin-qt} - -[ ! -x $BITCOIND ] && echo "$BITCOIND not found or not executable." && exit 1 - -# Don't allow man pages to be generated for binaries built from a dirty tree -DIRTY="" -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do - VERSION_OUTPUT=$($cmd --version) - if [[ $VERSION_OUTPUT == *"dirty"* ]]; then - DIRTY="${DIRTY}${cmd}\n" - fi -done -if [ -n "$DIRTY" ] -then - echo -e "WARNING: the following binaries were built from a dirty tree:\n" - echo -e $DIRTY - echo "man pages generated from dirty binaries should NOT be committed." - echo "To properly generate man pages, please commit your changes to the above binaries, rebuild them, then run this script again." -fi - -# The autodetected version git tag can screw up manpage output a little bit -read -r -a BTCVER <<< "$($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')" - -# Create a footer file with copyright content. -# This gets autodetected fine for bitcoind if --version-string is not set, -# but has different outcomes for bitcoin-qt and bitcoin-cli. -echo "[COPYRIGHT]" > footer.h2m -$BITCOIND --version | sed -n '1!p' >> footer.h2m - -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do - cmdname="${cmd##*/}" - help2man -N --version-string=${BTCVER[0]} --include=footer.h2m -o ${MANDIR}/${cmdname}.1 ${cmd} - sed -i "s/\\\-${BTCVER[1]}//g" ${MANDIR}/${cmdname}.1 -done - -rm -f footer.h2m diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp new file mode 100644 index 000000000000..919ffab102dd --- /dev/null +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -0,0 +1,7 @@ +# Fixups / upstreamed changes +[ + { include: [ "", private, "", public ] }, + { include: [ "", private, "", public ] }, + { include: [ "", private, "", public ] }, + { include: [ "", private, "", public ] }, +] diff --git a/contrib/devtools/pixie.py b/contrib/devtools/pixie.py deleted file mode 100644 index 64660968ad25..000000000000 --- a/contrib/devtools/pixie.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 Wladimir J. van der Laan -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Compact, self-contained ELF implementation for bitcoin-core security checks. -''' -import struct -import types -from typing import Dict, List, Optional, Union, Tuple - -# you can find all these values in elf.h -EI_NIDENT = 16 - -# Byte indices in e_ident -EI_CLASS = 4 # ELFCLASSxx -EI_DATA = 5 # ELFDATAxxxx - -ELFCLASS32 = 1 # 32-bit -ELFCLASS64 = 2 # 64-bit - -ELFDATA2LSB = 1 # little endian -ELFDATA2MSB = 2 # big endian - -# relevant values for e_machine -EM_386 = 3 -EM_PPC64 = 21 -EM_ARM = 40 -EM_AARCH64 = 183 -EM_X86_64 = 62 -EM_RISCV = 243 - -# relevant values for e_type -ET_DYN = 3 - -# relevant values for sh_type -SHT_PROGBITS = 1 -SHT_STRTAB = 3 -SHT_DYNAMIC = 6 -SHT_DYNSYM = 11 -SHT_GNU_verneed = 0x6ffffffe -SHT_GNU_versym = 0x6fffffff - -# relevant values for p_type -PT_LOAD = 1 -PT_GNU_STACK = 0x6474e551 -PT_GNU_RELRO = 0x6474e552 - -# relevant values for p_flags -PF_X = (1 << 0) -PF_W = (1 << 1) -PF_R = (1 << 2) - -# relevant values for d_tag -DT_NEEDED = 1 -DT_FLAGS = 30 - -# relevant values of `d_un.d_val' in the DT_FLAGS entry -DF_BIND_NOW = 0x00000008 - -# relevant d_tags with string payload -STRING_TAGS = {DT_NEEDED} - -# rrlevant values for ST_BIND subfield of st_info (symbol binding) -STB_LOCAL = 0 - -class ELFRecord(types.SimpleNamespace): - '''Unified parsing for ELF records.''' - def __init__(self, data: bytes, offset: int, eh: 'ELFHeader', total_size: Optional[int]) -> None: - hdr_struct = self.STRUCT[eh.ei_class][0][eh.ei_data] - if total_size is not None and hdr_struct.size > total_size: - raise ValueError(f'{self.__class__.__name__} header size too small ({total_size} < {hdr_struct.size})') - for field, value in zip(self.STRUCT[eh.ei_class][1], hdr_struct.unpack(data[offset:offset + hdr_struct.size])): - setattr(self, field, value) - -def BiStruct(chars: str) -> Dict[int, struct.Struct]: - '''Compile a struct parser for both endians.''' - return { - ELFDATA2LSB: struct.Struct('<' + chars), - ELFDATA2MSB: struct.Struct('>' + chars), - } - -class ELFHeader(ELFRecord): - FIELDS = ['e_type', 'e_machine', 'e_version', 'e_entry', 'e_phoff', 'e_shoff', 'e_flags', 'e_ehsize', 'e_phentsize', 'e_phnum', 'e_shentsize', 'e_shnum', 'e_shstrndx'] - STRUCT = { - ELFCLASS32: (BiStruct('HHIIIIIHHHHHH'), FIELDS), - ELFCLASS64: (BiStruct('HHIQQQIHHHHHH'), FIELDS), - } - - def __init__(self, data: bytes, offset: int) -> None: - self.e_ident = data[offset:offset + EI_NIDENT] - if self.e_ident[0:4] != b'\x7fELF': - raise ValueError('invalid ELF magic') - self.ei_class = self.e_ident[EI_CLASS] - self.ei_data = self.e_ident[EI_DATA] - - super().__init__(data, offset + EI_NIDENT, self, None) - - def __repr__(self) -> str: - return f'Header(e_ident={self.e_ident!r}, e_type={self.e_type}, e_machine={self.e_machine}, e_version={self.e_version}, e_entry={self.e_entry}, e_phoff={self.e_phoff}, e_shoff={self.e_shoff}, e_flags={self.e_flags}, e_ehsize={self.e_ehsize}, e_phentsize={self.e_phentsize}, e_phnum={self.e_phnum}, e_shentsize={self.e_shentsize}, e_shnum={self.e_shnum}, e_shstrndx={self.e_shstrndx})' - -class Section(ELFRecord): - name: Optional[bytes] = None - FIELDS = ['sh_name', 'sh_type', 'sh_flags', 'sh_addr', 'sh_offset', 'sh_size', 'sh_link', 'sh_info', 'sh_addralign', 'sh_entsize'] - STRUCT = { - ELFCLASS32: (BiStruct('IIIIIIIIII'), FIELDS), - ELFCLASS64: (BiStruct('IIQQQQIIQQ'), FIELDS), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, eh.e_shentsize) - self._data = data - - def __repr__(self) -> str: - return f'Section(sh_name={self.sh_name}({self.name!r}), sh_type=0x{self.sh_type:x}, sh_flags={self.sh_flags}, sh_addr=0x{self.sh_addr:x}, sh_offset=0x{self.sh_offset:x}, sh_size={self.sh_size}, sh_link={self.sh_link}, sh_info={self.sh_info}, sh_addralign={self.sh_addralign}, sh_entsize={self.sh_entsize})' - - def contents(self) -> bytes: - '''Return section contents.''' - return self._data[self.sh_offset:self.sh_offset + self.sh_size] - -class ProgramHeader(ELFRecord): - STRUCT = { - # different ELF classes have the same fields, but in a different order to optimize space versus alignment - ELFCLASS32: (BiStruct('IIIIIIII'), ['p_type', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_flags', 'p_align']), - ELFCLASS64: (BiStruct('IIQQQQQQ'), ['p_type', 'p_flags', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_align']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, eh.e_phentsize) - - def __repr__(self) -> str: - return f'ProgramHeader(p_type={self.p_type}, p_offset={self.p_offset}, p_vaddr={self.p_vaddr}, p_paddr={self.p_paddr}, p_filesz={self.p_filesz}, p_memsz={self.p_memsz}, p_flags={self.p_flags}, p_align={self.p_align})' - -class Symbol(ELFRecord): - STRUCT = { - # different ELF classes have the same fields, but in a different order to optimize space versus alignment - ELFCLASS32: (BiStruct('IIIBBH'), ['st_name', 'st_value', 'st_size', 'st_info', 'st_other', 'st_shndx']), - ELFCLASS64: (BiStruct('IBBHQQ'), ['st_name', 'st_info', 'st_other', 'st_shndx', 'st_value', 'st_size']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, symtab: Section, strings: bytes, version: Optional[bytes]) -> None: - super().__init__(data, offset, eh, symtab.sh_entsize) - self.name = _lookup_string(strings, self.st_name) - self.version = version - - def __repr__(self) -> str: - return f'Symbol(st_name={self.st_name}({self.name!r}), st_value={self.st_value}, st_size={self.st_size}, st_info={self.st_info}, st_other={self.st_other}, st_shndx={self.st_shndx}, version={self.version!r})' - - @property - def is_import(self) -> bool: - '''Returns whether the symbol is an imported symbol.''' - return self.st_bind != STB_LOCAL and self.st_shndx == 0 - - @property - def is_export(self) -> bool: - '''Returns whether the symbol is an exported symbol.''' - return self.st_bind != STB_LOCAL and self.st_shndx != 0 - - @property - def st_bind(self) -> int: - '''Returns STB_*.''' - return self.st_info >> 4 - -class Verneed(ELFRecord): - DEF = (BiStruct('HHIII'), ['vn_version', 'vn_cnt', 'vn_file', 'vn_aux', 'vn_next']) - STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, None) - - def __repr__(self) -> str: - return f'Verneed(vn_version={self.vn_version}, vn_cnt={self.vn_cnt}, vn_file={self.vn_file}, vn_aux={self.vn_aux}, vn_next={self.vn_next})' - -class Vernaux(ELFRecord): - DEF = (BiStruct('IHHII'), ['vna_hash', 'vna_flags', 'vna_other', 'vna_name', 'vna_next']) - STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, strings: bytes) -> None: - super().__init__(data, offset, eh, None) - self.name = _lookup_string(strings, self.vna_name) - - def __repr__(self) -> str: - return f'Veraux(vna_hash={self.vna_hash}, vna_flags={self.vna_flags}, vna_other={self.vna_other}, vna_name={self.vna_name}({self.name!r}), vna_next={self.vna_next})' - -class DynTag(ELFRecord): - STRUCT = { - ELFCLASS32: (BiStruct('II'), ['d_tag', 'd_val']), - ELFCLASS64: (BiStruct('QQ'), ['d_tag', 'd_val']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, section: Section) -> None: - super().__init__(data, offset, eh, section.sh_entsize) - - def __repr__(self) -> str: - return f'DynTag(d_tag={self.d_tag}, d_val={self.d_val})' - -def _lookup_string(data: bytes, index: int) -> bytes: - '''Look up string by offset in ELF string table.''' - endx = data.find(b'\x00', index) - assert endx != -1 - return data[index:endx] - -VERSYM_S = BiStruct('H') # .gnu_version section has a single 16-bit integer per symbol in the linked section -def _parse_symbol_table(section: Section, strings: bytes, eh: ELFHeader, versym: bytes, verneed: Dict[int, bytes]) -> List[Symbol]: - '''Parse symbol table, return a list of symbols.''' - data = section.contents() - symbols = [] - versym_iter = (verneed.get(v[0]) for v in VERSYM_S[eh.ei_data].iter_unpack(versym)) - for ofs, version in zip(range(0, len(data), section.sh_entsize), versym_iter): - symbols.append(Symbol(data, ofs, eh, section, strings, version)) - return symbols - -def _parse_verneed(section: Section, strings: bytes, eh: ELFHeader) -> Dict[int, bytes]: - '''Parse .gnu.version_r section, return a dictionary of {versym: 'GLIBC_...'}.''' - data = section.contents() - ofs = 0 - result = {} - while True: - verneed = Verneed(data, ofs, eh) - aofs = ofs + verneed.vn_aux - while True: - vernaux = Vernaux(data, aofs, eh, strings) - result[vernaux.vna_other] = vernaux.name - if not vernaux.vna_next: - break - aofs += vernaux.vna_next - - if not verneed.vn_next: - break - ofs += verneed.vn_next - - return result - -def _parse_dyn_tags(section: Section, strings: bytes, eh: ELFHeader) -> List[Tuple[int, Union[bytes, int]]]: - '''Parse dynamic tags. Return array of tuples.''' - data = section.contents() - ofs = 0 - result = [] - for ofs in range(0, len(data), section.sh_entsize): - tag = DynTag(data, ofs, eh, section) - val = _lookup_string(strings, tag.d_val) if tag.d_tag in STRING_TAGS else tag.d_val - result.append((tag.d_tag, val)) - - return result - -class ELFFile: - sections: List[Section] - program_headers: List[ProgramHeader] - dyn_symbols: List[Symbol] - dyn_tags: List[Tuple[int, Union[bytes, int]]] - - def __init__(self, data: bytes) -> None: - self.data = data - self.hdr = ELFHeader(self.data, 0) - self._load_sections() - self._load_program_headers() - self._load_dyn_symbols() - self._load_dyn_tags() - self._section_to_segment_mapping() - - def _load_sections(self) -> None: - self.sections = [] - for idx in range(self.hdr.e_shnum): - offset = self.hdr.e_shoff + idx * self.hdr.e_shentsize - self.sections.append(Section(self.data, offset, self.hdr)) - - shstr = self.sections[self.hdr.e_shstrndx].contents() - for section in self.sections: - section.name = _lookup_string(shstr, section.sh_name) - - def _load_program_headers(self) -> None: - self.program_headers = [] - for idx in range(self.hdr.e_phnum): - offset = self.hdr.e_phoff + idx * self.hdr.e_phentsize - self.program_headers.append(ProgramHeader(self.data, offset, self.hdr)) - - def _load_dyn_symbols(self) -> None: - # first, load 'verneed' section - verneed = None - for section in self.sections: - if section.sh_type == SHT_GNU_verneed: - strtab = self.sections[section.sh_link].contents() # associated string table - assert verneed is None # only one section of this kind please - verneed = _parse_verneed(section, strtab, self.hdr) - assert verneed is not None - - # then, correlate GNU versym sections with dynamic symbol sections - versym = {} - for section in self.sections: - if section.sh_type == SHT_GNU_versym: - versym[section.sh_link] = section - - # finally, load dynsym sections - self.dyn_symbols = [] - for idx, section in enumerate(self.sections): - if section.sh_type == SHT_DYNSYM: # find dynamic symbol tables - strtab_data = self.sections[section.sh_link].contents() # associated string table - versym_data = versym[idx].contents() # associated symbol version table - self.dyn_symbols += _parse_symbol_table(section, strtab_data, self.hdr, versym_data, verneed) - - def _load_dyn_tags(self) -> None: - self.dyn_tags = [] - for idx, section in enumerate(self.sections): - if section.sh_type == SHT_DYNAMIC: # find dynamic tag tables - strtab = self.sections[section.sh_link].contents() # associated string table - self.dyn_tags += _parse_dyn_tags(section, strtab, self.hdr) - - def _section_to_segment_mapping(self) -> None: - for ph in self.program_headers: - ph.sections = [] - for section in self.sections: - if ph.p_vaddr <= section.sh_addr < (ph.p_vaddr + ph.p_memsz): - ph.sections.append(section) - - def query_dyn_tags(self, tag_in: int) -> List[Union[int, bytes]]: - '''Return the values of all dyn tags with the specified tag.''' - return [val for (tag, val) in self.dyn_tags if tag == tag_in] - - -def load(filename: str) -> ELFFile: - with open(filename, 'rb') as f: - data = f.read() - return ELFFile(data) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 0b59d8eada68..05c0af029ec8 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -8,248 +8,250 @@ Otherwise the exit status will be 1 and it will log which executables failed which checks. ''' import sys -from typing import List, Optional +from typing import List -import lief -import pixie +import lief #type:ignore -def check_ELF_PIE(executable) -> bool: - ''' - Check for position independent executable (PIE), allowing for address space randomization. - ''' - elf = pixie.load(executable) - return elf.hdr.e_type == pixie.ET_DYN - -def check_ELF_NX(executable) -> bool: - ''' - Check that no sections are writable and executable (including the stack) - ''' - elf = pixie.load(executable) - have_wx = False - have_gnu_stack = False - for ph in elf.program_headers: - if ph.p_type == pixie.PT_GNU_STACK: - have_gnu_stack = True - if (ph.p_flags & pixie.PF_W) != 0 and (ph.p_flags & pixie.PF_X) != 0: # section is both writable and executable - have_wx = True - return have_gnu_stack and not have_wx - -def check_ELF_RELRO(executable) -> bool: +def check_ELF_RELRO(binary) -> bool: ''' Check for read-only relocations. GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag ''' - elf = pixie.load(executable) have_gnu_relro = False - for ph in elf.program_headers: + for segment in binary.segments: # Note: not checking p_flags == PF_R: here as linkers set the permission differently # This does not affect security: the permission flags of the GNU_RELRO program # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also https://marc.info/?l=binutils&m=1498883354122353 - if ph.p_type == pixie.PT_GNU_RELRO: + if segment.type == lief.ELF.SEGMENT_TYPES.GNU_RELRO: have_gnu_relro = True have_bindnow = False - for flags in elf.query_dyn_tags(pixie.DT_FLAGS): - assert isinstance(flags, int) - if flags & pixie.DF_BIND_NOW: + try: + flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS) + if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW: have_bindnow = True + except: + have_bindnow = False return have_gnu_relro and have_bindnow -def check_ELF_Canary(executable) -> bool: +def check_ELF_Canary(binary) -> bool: ''' Check for use of stack canary ''' - elf = pixie.load(executable) - ok = False - for symbol in elf.dyn_symbols: - if symbol.name == b'__stack_chk_fail': - ok = True - return ok + return binary.has_symbol('__stack_chk_fail') -def check_ELF_separate_code(executable): +def check_ELF_separate_code(binary): ''' Check that sections are appropriately separated in virtual memory, based on their permissions. This checks for missing -Wl,-z,separate-code and potentially other problems. ''' - elf = pixie.load(executable) - R = pixie.PF_R - W = pixie.PF_W - E = pixie.PF_X + R = lief.ELF.SEGMENT_FLAGS.R + W = lief.ELF.SEGMENT_FLAGS.W + E = lief.ELF.SEGMENT_FLAGS.X EXPECTED_FLAGS = { # Read + execute - b'.init': R | E, - b'.plt': R | E, - b'.plt.got': R | E, - b'.plt.sec': R | E, - b'.text': R | E, - b'.fini': R | E, + '.init': R | E, + '.plt': R | E, + '.plt.got': R | E, + '.plt.sec': R | E, + '.text': R | E, + '.fini': R | E, # Read-only data - b'.interp': R, - b'.note.gnu.property': R, - b'.note.gnu.build-id': R, - b'.note.ABI-tag': R, - b'.gnu.hash': R, - b'.dynsym': R, - b'.dynstr': R, - b'.gnu.version': R, - b'.gnu.version_r': R, - b'.rela.dyn': R, - b'.rela.plt': R, - b'.rodata': R, - b'.eh_frame_hdr': R, - b'.eh_frame': R, - b'.qtmetadata': R, - b'.gcc_except_table': R, - b'.stapsdt.base': R, + '.interp': R, + '.note.gnu.property': R, + '.note.gnu.build-id': R, + '.note.ABI-tag': R, + '.gnu.hash': R, + '.dynsym': R, + '.dynstr': R, + '.gnu.version': R, + '.gnu.version_r': R, + '.rela.dyn': R, + '.rela.plt': R, + '.rodata': R, + '.eh_frame_hdr': R, + '.eh_frame': R, + '.qtmetadata': R, + '.gcc_except_table': R, + '.stapsdt.base': R, # Writable data - b'.init_array': R | W, - b'.fini_array': R | W, - b'.dynamic': R | W, - b'.got': R | W, - b'.data': R | W, - b'.bss': R | W, + '.init_array': R | W, + '.fini_array': R | W, + '.dynamic': R | W, + '.got': R | W, + '.data': R | W, + '.bss': R | W, } - if elf.hdr.e_machine == pixie.EM_PPC64: + if binary.header.machine_type == lief.ELF.ARCH.PPC64: # .plt is RW on ppc64 even with separate-code - EXPECTED_FLAGS[b'.plt'] = R | W + EXPECTED_FLAGS['.plt'] = R | W # For all LOAD program headers get mapping to the list of sections, # and for each section, remember the flags of the associated program header. flags_per_section = {} - for ph in elf.program_headers: - if ph.p_type == pixie.PT_LOAD: - for section in ph.sections: - assert(section.name not in flags_per_section) - flags_per_section[section.name] = ph.p_flags + for segment in binary.segments: + if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: + for section in segment.sections: + flags_per_section[section.name] = segment.flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags for (section, flags) in flags_per_section.items(): if section in EXPECTED_FLAGS: - if EXPECTED_FLAGS[section] != flags: + if int(EXPECTED_FLAGS[section]) != int(flags): return False return True -def check_PE_DYNAMIC_BASE(executable) -> bool: +def check_ELF_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_function_address('main') + content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_PE_DYNAMIC_BASE(binary) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - binary = lief.parse(executable) return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists # Must support high-entropy 64-bit address space layout randomization # in addition to DYNAMIC_BASE to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(executable) -> bool: +def check_PE_HIGH_ENTROPY_VA(binary) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - binary = lief.parse(executable) return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists -def check_PE_RELOC_SECTION(executable) -> bool: +def check_PE_RELOC_SECTION(binary) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' - binary = lief.parse(executable) return binary.has_relocations -def check_MACHO_NOUNDEFS(executable) -> bool: +def check_PE_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_symbol('main').value + + section_addr = binary.section_from_rva(main).virtual_address + virtual_address = binary.optional_header.imagebase + section_addr + main + + content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_MACHO_NOUNDEFS(binary) -> bool: ''' Check for no undefined references. ''' - binary = lief.parse(executable) return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) -def check_MACHO_LAZY_BINDINGS(executable) -> bool: +def check_MACHO_LAZY_BINDINGS(binary) -> bool: ''' Check for no lazy bindings. We don't use or check for MH_BINDATLOAD. See #18295. ''' - binary = lief.parse(executable) return binary.dyld_info.lazy_bind == (0,0) -def check_MACHO_Canary(executable) -> bool: +def check_MACHO_Canary(binary) -> bool: ''' Check for use of stack canary ''' - binary = lief.parse(executable) return binary.has_symbol('___stack_chk_fail') -def check_PIE(executable) -> bool: +def check_PIE(binary) -> bool: ''' Check for position independent executable (PIE), allowing for address space randomization. ''' - binary = lief.parse(executable) return binary.is_pie -def check_NX(executable) -> bool: +def check_NX(binary) -> bool: ''' Check for no stack execution ''' - binary = lief.parse(executable) return binary.has_nx -def check_control_flow(executable) -> bool: +def check_MACHO_control_flow(binary) -> bool: ''' Check for control flow instrumentation ''' - binary = lief.parse(executable) - content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) if content == [243, 15, 30, 250]: # endbr64 return True return False - -CHECKS = { -'ELF': [ - ('PIE', check_ELF_PIE), - ('NX', check_ELF_NX), +BASE_ELF = [ + ('PIE', check_PIE), + ('NX', check_NX), ('RELRO', check_ELF_RELRO), ('Canary', check_ELF_Canary), ('separate_code', check_ELF_separate_code), -], -'PE': [ +] + +BASE_PE = [ ('PIE', check_PIE), ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION) -], -'MACHO': [ - ('PIE', check_PIE), + ('RELOC_SECTION', check_PE_RELOC_SECTION), + ('CONTROL_FLOW', check_PE_control_flow), +] + +BASE_MACHO = [ ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('NX', check_NX), ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), ('Canary', check_MACHO_Canary), - ('CONTROL_FLOW', check_control_flow), ] -} -def identify_executable(executable) -> Optional[str]: - with open(filename, 'rb') as f: - magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' - return None +CHECKS = { + lief.EXE_FORMATS.ELF: { + lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], + lief.ARCHITECTURES.ARM: BASE_ELF, + lief.ARCHITECTURES.ARM64: BASE_ELF, + lief.ARCHITECTURES.PPC: BASE_ELF, + lief.ARCHITECTURES.RISCV: BASE_ELF, + }, + lief.EXE_FORMATS.PE: { + lief.ARCHITECTURES.X86: BASE_PE, + }, + lief.EXE_FORMATS.MACHO: { + lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE), + ('NX', check_NX), + ('CONTROL_FLOW', check_MACHO_control_flow)], + lief.ARCHITECTURES.ARM64: BASE_MACHO, + } +} if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: try: - etype = identify_executable(filename) - if etype is None: - print(f'{filename}: unknown format') + binary = lief.parse(filename) + etype = binary.format + arch = binary.abstract.header.architecture + binary.concrete + + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') + retval = 1 + continue + + if arch == lief.ARCHITECTURES.NONE: + print(f'{filename}: unknown architecture') retval = 1 continue failed: List[str] = [] - for (name, func) in CHECKS[etype]: - if not func(filename): + for (name, func) in CHECKS[etype][arch]: + if not func(binary): failed.append(name) if failed: print(f'{filename}: failed {" ".join(failed)}') diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 61f727fa6342..4b1cceb57ceb 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -10,56 +10,66 @@ find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' -import subprocess import sys -from typing import List, Optional +from typing import List, Dict -import lief -import pixie +import lief #type:ignore -from utils import determine_wellknown_cmd - -# Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases -# -# - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.19 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=libc6) -# -# Ubuntu 16.04 (Xenial) EOL: 2024. https://wiki.ubuntu.com/Releases +# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases # -# - g++ version 5.3.1 (https://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=xenial§ion=all) -# - libc version 2.23.0 (https://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=xenial§ion=all) +# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) # -# CentOS 7 EOL: 2024. https://wiki.centos.org/FAQ/General +# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases # -# - g++ version 4.8.5 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) -# - libc version 2.17 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) +# - g++ version 5.3.1 +# - libc version 2.23 # -# Taking the minimum of these as our target. +# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product # -# According to GNU ABI document (https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: -# GCC 4.8.5: GCC_4.8.0 -# (glibc) GLIBC_2_17 +# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # +# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. + MAX_VERSIONS = { 'GCC': (4,8,0), 'GLIBC': { - pixie.EM_386: (2,17), - pixie.EM_X86_64: (2,17), - pixie.EM_ARM: (2,17), - pixie.EM_AARCH64:(2,17), - pixie.EM_PPC64: (2,17), - pixie.EM_RISCV: (2,27), + lief.ELF.ARCH.x86_64: (2,18), + lief.ELF.ARCH.ARM: (2,18), + lief.ELF.ARCH.AARCH64:(2,18), + lief.ELF.ARCH.PPC64: (2,18), + lief.ELF.ARCH.RISCV: (2,27), }, 'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) } -# See here for a description of _IO_stdin_used: -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', -'environ', '_environ', '__environ', +'environ', '_environ', '__environ', '_fini', '_init', 'stdin', +'stdout', 'stderr', +} + +# Expected linker-loader names can be found here: +# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 +ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { + lief.ELF.ARCH.x86_64: { + lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + }, + lief.ELF.ARCH.ARM: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", + }, + lief.ELF.ARCH.AARCH64: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", + }, + lief.ELF.ARCH.PPC64: { + lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", + lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", + }, + lief.ELF.ARCH.RISCV: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", + }, } # Allowed NEEDED libraries @@ -84,7 +94,19 @@ 'libxkbcommon-x11.so.0', # keyboard keymapping 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing -'libdl.so.2' # programming interface to dynamic linker +'libdl.so.2', # programming interface to dynamic linker +'libxcb-icccm.so.4', +'libxcb-image.so.0', +'libxcb-shm.so.0', +'libxcb-keysyms.so.1', +'libxcb-randr.so.0', +'libxcb-render-util.so.0', +'libxcb-render.so.0', +'libxcb-shape.so.0', +'libxcb-sync.so.1', +'libxcb-xfixes.so.0', +'libxcb-xinerama.so.0', +'libxcb-xkb.so.1', } MACHO_ALLOWED_LIBRARIES = { @@ -95,6 +117,7 @@ 'AppKit', # user interface 'ApplicationServices', # common application tasks. 'Carbon', # deprecated c back-compat API +'ColorSync', 'CoreFoundation', # low level func, data types 'CoreGraphics', # 2D rendering 'CoreServices', # operating system services @@ -133,31 +156,8 @@ 'WTSAPI32.dll', } -class CPPFilt(object): - ''' - Demangle C++ symbol names. - - Use a pipe to the 'c++filt' command. - ''' - def __init__(self): - self.proc = subprocess.Popen(determine_wellknown_cmd('CPPFILT', 'c++filt'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - - def __call__(self, mangled): - self.proc.stdin.write(mangled + '\n') - self.proc.stdin.flush() - return self.proc.stdout.readline().rstrip() - - def close(self): - self.proc.stdin.close() - self.proc.stdout.close() - self.proc.wait() - def check_version(max_versions, version, arch) -> bool: - if '_' in version: - (lib, _, ver) = version.rpartition('_') - else: - lib = version - ver = '0' + (lib, _, ver) = version.rpartition('_') ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False @@ -166,48 +166,45 @@ def check_version(max_versions, version, arch) -> bool: else: return ver <= max_versions[lib][arch] -def check_imported_symbols(filename) -> bool: - elf = pixie.load(filename) - cppfilt = CPPFilt() +def check_imported_symbols(binary) -> bool: ok: bool = True - for symbol in elf.dyn_symbols: - if not symbol.is_import: + for symbol in binary.imported_symbols: + if not symbol.imported: continue - sym = symbol.name.decode() - version = symbol.version.decode() if symbol.version is not None else None - if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine): - print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version)) - ok = False + + version = symbol.symbol_version if symbol.has_version else None + + if version: + aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None + if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): + print(f'{filename}: symbol {symbol.name} from unsupported version {version}') + ok = False return ok -def check_exported_symbols(filename) -> bool: - elf = pixie.load(filename) - cppfilt = CPPFilt() +def check_exported_symbols(binary) -> bool: ok: bool = True - for symbol in elf.dyn_symbols: - if not symbol.is_export: + + for symbol in binary.dynamic_symbols: + if not symbol.exported: continue - sym = symbol.name.decode() - if elf.hdr.e_machine == pixie.EM_RISCV or sym in IGNORE_EXPORTS: + name = symbol.name + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: continue - print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym))) + print(f'{binary.name}: export of symbol {name} not allowed!') ok = False return ok -def check_ELF_libraries(filename) -> bool: +def check_ELF_libraries(binary) -> bool: ok: bool = True - elf = pixie.load(filename) - for library_name in elf.query_dyn_tags(pixie.DT_NEEDED): - assert(isinstance(library_name, bytes)) - if library_name.decode() not in ELF_ALLOWED_LIBRARIES: - print('{}: NEEDED library {} is not allowed'.format(filename, library_name.decode())) + for library in binary.libraries: + if library not in ELF_ALLOWED_LIBRARIES: + print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') ok = False return ok -def check_MACHO_libraries(filename) -> bool: +def check_MACHO_libraries(binary) -> bool: ok: bool = True - binary = lief.parse(filename) for dylib in binary.libraries: split = dylib.name.split('/') if split[-1] not in MACHO_ALLOWED_LIBRARIES: @@ -215,76 +212,68 @@ def check_MACHO_libraries(filename) -> bool: ok = False return ok -def check_MACHO_min_os(filename) -> bool: - binary = lief.parse(filename) - if binary.build_version.minos == [10,14,0]: +def check_MACHO_min_os(binary) -> bool: + if binary.build_version.minos == [10,15,0]: return True return False -def check_MACHO_sdk(filename) -> bool: - binary = lief.parse(filename) - if binary.build_version.sdk == [10, 15, 6]: +def check_MACHO_sdk(binary) -> bool: + if binary.build_version.sdk == [11, 0, 0]: return True return False -def check_PE_libraries(filename) -> bool: +def check_PE_libraries(binary) -> bool: ok: bool = True - binary = lief.parse(filename) for dylib in binary.libraries: if dylib not in PE_ALLOWED_LIBRARIES: print(f'{dylib} is not in ALLOWED_LIBRARIES!') ok = False return ok -def check_PE_subsystem_version(filename) -> bool: - binary = lief.parse(filename) +def check_PE_subsystem_version(binary) -> bool: major: int = binary.optional_header.major_subsystem_version minor: int = binary.optional_header.minor_subsystem_version if major == 6 and minor == 1: return True return False +def check_ELF_interpreter(binary) -> bool: + expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] + + return binary.concrete.interpreter == expected_interpreter + CHECKS = { -'ELF': [ +lief.EXE_FORMATS.ELF: [ ('IMPORTED_SYMBOLS', check_imported_symbols), ('EXPORTED_SYMBOLS', check_exported_symbols), - ('LIBRARY_DEPENDENCIES', check_ELF_libraries) + ('LIBRARY_DEPENDENCIES', check_ELF_libraries), + ('INTERPRETER_NAME', check_ELF_interpreter), ], -'MACHO': [ +lief.EXE_FORMATS.MACHO: [ ('DYNAMIC_LIBRARIES', check_MACHO_libraries), ('MIN_OS', check_MACHO_min_os), ('SDK', check_MACHO_sdk), ], -'PE' : [ +lief.EXE_FORMATS.PE: [ ('DYNAMIC_LIBRARIES', check_PE_libraries), ('SUBSYSTEM_VERSION', check_PE_subsystem_version), ] } -def identify_executable(executable) -> Optional[str]: - with open(filename, 'rb') as f: - magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' - return None - if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: try: - etype = identify_executable(filename) - if etype is None: - print(f'{filename}: unknown format') + binary = lief.parse(filename) + etype = binary.format + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') retval = 1 continue failed: List[str] = [] for (name, func) in CHECKS[etype]: - if not func(filename): + if not func(binary): failed.append(name) if failed: print(f'{filename}: failed {" ".join(failed)}') diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 14058e2cc8d5..84283e3522b8 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Test script for security-check.py ''' +import lief #type:ignore import os import subprocess +from typing import List import unittest from utils import determine_wellknown_cmd @@ -27,29 +29,62 @@ def clean_files(source, executable): os.remove(executable) def call_security_check(cc, source, executable, options): - subprocess.run([*cc,source,'-o',executable] + options, check=True) - p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) + p = subprocess.run([os.path.join(os.path.dirname(__file__), 'security-check.py'), executable], stdout=subprocess.PIPE, universal_newlines=True) return (p.returncode, p.stdout.rstrip()) +def get_arch(cc, source, executable): + subprocess.run([*cc, source, '-o', executable], check=True) + binary = lief.parse(executable) + arch = binary.abstract.header.architecture + os.remove(executable) + return arch + class TestSecurityChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' cc = determine_wellknown_cmd('CC', 'gcc') write_testcode(source) + arch = get_arch(cc, source, executable) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), - (1, executable+': failed separate_code')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), - (0, '')) + if arch == lief.ARCHITECTURES.X86: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full']), + (0, '')) + else: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (0, '')) clean_files(source, executable) @@ -59,17 +94,19 @@ def test_PE(self): cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing unless --dynamicbase is also supplied - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), (0, '')) clean_files(source, executable) @@ -79,21 +116,34 @@ def test_MACHO(self): executable = 'test1' cc = determine_wellknown_cmd('CC', 'clang') write_testcode(source) + arch = get_arch(cc, source, executable) + + if arch == lief.ARCHITECTURES.X86: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), + (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), + (1, executable+': failed PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (1, executable+': failed PIE')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (0, '')) + else: + # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']), + (1, executable+': failed LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']), + (0, '')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), - (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), - (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), - (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), - (1, executable+': failed PIE LAZY_BINDINGS CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), - (1, executable+': failed PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), - (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), - (0, '')) clean_files(source, executable) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 7d83c5f751f0..85d3a6d07bab 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -13,8 +13,17 @@ from utils import determine_wellknown_cmd def call_symbol_check(cc: List[str], source, executable, options): - subprocess.run([*cc,source,'-o',executable] + options, check=True) - p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) + p = subprocess.run([os.path.join(os.path.dirname(__file__), 'symbol-check.py'), executable], stdout=subprocess.PIPE, universal_newlines=True) os.remove(source) os.remove(executable) return (p.returncode, p.stdout.rstrip()) @@ -30,12 +39,12 @@ def test_ELF(self): cc = determine_wellknown_cmd('CC', 'gcc') # there's no way to do this test for RISC-V at the moment; we build for - # RISC-V in a glibc 2.27 envinonment and we allow all symbols from 2.27. + # RISC-V in a glibc 2.27 environment and we allow all symbols from 2.27. if 'riscv' in get_machine(cc): self.skipTest("test not available for RISC-V") # nextup was introduced in GLIBC 2.24, so is newer than our supported - # glibc (2.17), and available in our release build environment (2.24). + # glibc (2.18), and available in our release build environment (2.24). with open(source, 'w', encoding="utf8") as f: f.write(''' #define _GNU_SOURCE @@ -51,7 +60,7 @@ def test_ELF(self): ''') self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), - (1, executable + ': symbol nextup from unsupported version GLIBC_2.24\n' + + (1, executable + ': symbol nextup from unsupported version GLIBC_2.24(3)\n' + executable + ': failed IMPORTED_SYMBOLS')) # -lutil is part of the libc6 package so a safe bet that it's installed @@ -70,23 +79,24 @@ def test_ELF(self): ''') self.assertEqual(call_symbol_check(cc, source, executable, ['-lutil']), - (1, executable + ': NEEDED library libutil.so.1 is not allowed\n' + + (1, executable + ': libutil.so.1 is not in ALLOWED_LIBRARIES!\n' + executable + ': failed LIBRARY_DEPENDENCIES')) - # finally, check a conforming file that simply uses a math function + # finally, check a simple conforming binary source = 'test3.c' executable = 'test3' with open(source, 'w', encoding="utf8") as f: f.write(''' - #include + #include int main() { - return (int)pow(2.0, 4.0); + printf("42"); + return 0; } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), + self.assertEqual(call_symbol_check(cc, source, executable, []), (0, '')) def test_MACHO(self): @@ -136,7 +146,7 @@ def test_MACHO(self): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.14', '-Wl,11.4']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.15', '-Wl,11.4']), (1, f'{executable}: failed SDK')) def test_PE(self): @@ -177,7 +187,7 @@ def test_PE(self): executable = 'test3.exe' with open(source, 'w', encoding="utf8") as f: f.write(''' - #include + #include int main() { diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py deleted file mode 100755 index 5df87d9e70cb..000000000000 --- a/contrib/gitian-build.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -import argparse -import os -import subprocess -import sys - -def setup(): - global args, workdir - programs = ['ruby', 'git', 'make', 'wget', 'curl'] - if args.kvm: - programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils'] - elif args.docker: - if not os.path.isfile('/lib/systemd/system/docker.service'): - dockers = ['docker.io', 'docker-ce'] - for i in dockers: - return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) - if return_code == 0: - break - if return_code != 0: - print('Cannot find any way to install Docker.', file=sys.stderr) - sys.exit(1) - else: - programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] - subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) - if not os.path.isdir('gitian.sigs'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/gitian.sigs.git']) - if not os.path.isdir('bitcoin-detached-sigs'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/bitcoin-detached-sigs.git']) - if not os.path.isdir('gitian-builder'): - subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) - if not os.path.isdir('bitcoin'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin/bitcoin.git']) - os.chdir('gitian-builder') - make_image_prog = ['bin/make-base-vm', '--suite', 'focal', '--arch', 'amd64'] - if args.docker: - make_image_prog += ['--docker'] - elif not args.kvm: - make_image_prog += ['--lxc', '--disksize', '13000'] - subprocess.check_call(make_image_prog) - os.chdir(workdir) - if args.is_focal and not args.kvm and not args.docker: - subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) - print('Reboot is required') - sys.exit(0) - -def build(): - global args, workdir - - os.makedirs('bitcoin-binaries/' + args.version, exist_ok=True) - print('\nBuilding Dependencies\n') - os.chdir('gitian-builder') - os.makedirs('inputs', exist_ok=True) - - subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz']) - subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True) - subprocess.check_call(['make', '-C', '../bitcoin/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) - - if args.linux: - print('\nCompiling ' + args.version + ' Linux') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) - subprocess.check_call('mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) - - if args.windows: - print('\nCompiling ' + args.version + ' Windows') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) - subprocess.check_call('mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/', shell=True) - subprocess.check_call('mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) - - if args.macos: - print('\nCompiling ' + args.version + ' MacOS') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) - subprocess.check_call('mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/', shell=True) - subprocess.check_call('mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) - - os.chdir(workdir) - - if args.commit_files: - print('\nCommitting '+args.version+' Unsigned Sigs\n') - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-win-unsigned/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-osx-unsigned/'+args.signer]) - subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) - os.chdir(workdir) - -def sign(): - global args, workdir - os.chdir('gitian-builder') - - if args.windows: - print('\nSigning ' + args.version + ' Windows') - subprocess.check_call('cp inputs/bitcoin-' + args.version + '-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz', shell=True) - subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) - subprocess.check_call('mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/'+args.version, shell=True) - - if args.macos: - print('\nSigning ' + args.version + ' MacOS') - subprocess.check_call('cp inputs/bitcoin-' + args.version + '-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz', shell=True) - subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) - subprocess.check_call('mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/'+args.version+'/bitcoin-'+args.version+'-osx.dmg', shell=True) - - os.chdir(workdir) - - if args.commit_files: - print('\nCommitting '+args.version+' Signed Sigs\n') - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'add', args.version+'-win-signed/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-osx-signed/'+args.signer]) - subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer]) - os.chdir(workdir) - -def verify(): - global args, workdir - rc = 0 - os.chdir('gitian-builder') - - print('\nVerifying v'+args.version+' Linux\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']): - print('Verifying v'+args.version+' Linux FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' Windows\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']): - print('Verifying v'+args.version+' Windows FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' MacOS\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']): - print('Verifying v'+args.version+' MacOS FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' Signed Windows\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']): - print('Verifying v'+args.version+' Signed Windows FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' Signed MacOS\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']): - print('Verifying v'+args.version+' Signed MacOS FAILED\n') - rc = 1 - - os.chdir(workdir) - return rc - -def main(): - global args, workdir - - parser = argparse.ArgumentParser(description='Script for running full Gitian builds.') - parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') - parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') - parser.add_argument('-u', '--url', dest='url', default='https://github.com/bitcoin/bitcoin', help='Specify the URL of the repository. Default is %(default)s') - parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') - parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') - parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS') - parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') - parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') - parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') - parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') - parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') - parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') - parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)') - parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') - parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') - parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file') - parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') - - args = parser.parse_args() - workdir = os.getcwd() - - args.is_focal = b'focal' in subprocess.check_output(['lsb_release', '-cs']) - - if args.kvm and args.docker: - raise Exception('Error: cannot have both kvm and docker') - - # Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they - # can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm). - os.environ['USE_LXC'] = '' - os.environ['USE_VBOX'] = '' - os.environ['USE_DOCKER'] = '' - if args.docker: - os.environ['USE_DOCKER'] = '1' - elif not args.kvm: - os.environ['USE_LXC'] = '1' - if 'GITIAN_HOST_IP' not in os.environ.keys(): - os.environ['GITIAN_HOST_IP'] = '10.0.3.1' - if 'LXC_GUEST_IP' not in os.environ.keys(): - os.environ['LXC_GUEST_IP'] = '10.0.3.5' - - if args.setup: - setup() - - if args.buildsign: - args.build = True - args.sign = True - - if not args.build and not args.sign and not args.verify: - sys.exit(0) - - args.linux = 'l' in args.os - args.windows = 'w' in args.os - args.macos = 'm' in args.os - - # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'): - print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') - args.macos = False - - args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' - - script_name = os.path.basename(sys.argv[0]) - if not args.signer: - print(script_name+': Missing signer') - print('Try '+script_name+' --help for more information') - sys.exit(1) - if not args.version: - print(script_name+': Missing version') - print('Try '+script_name+' --help for more information') - sys.exit(1) - - # Add leading 'v' for tags - if args.commit and args.pull: - raise Exception('Cannot have both commit and pull') - args.commit = ('' if args.commit else 'v') + args.version - - os.chdir('bitcoin') - if args.pull: - subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) - os.chdir('../gitian-builder/inputs/bitcoin') - subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) - args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip() - args.version = 'pull-' + args.version - print(args.commit) - subprocess.check_call(['git', 'fetch']) - subprocess.check_call(['git', 'checkout', args.commit]) - os.chdir(workdir) - - os.chdir('gitian-builder') - subprocess.check_call(['git', 'pull']) - os.chdir(workdir) - - if args.build: - build() - - if args.sign: - sign() - - if args.verify: - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'pull']) - os.chdir(workdir) - sys.exit(verify()) - -if __name__ == '__main__': - main() diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME deleted file mode 100644 index 330fbc041b31..000000000000 --- a/contrib/gitian-descriptors/assign_DISTNAME +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# A helper script to be sourced into the gitian descriptors - -if RECENT_TAG="$(git describe --exact-match HEAD 2> /dev/null)"; then - VERSION="${RECENT_TAG#v}" -else - VERSION="$(git rev-parse --short=12 HEAD)" -fi -DISTNAME="bitcoin-${VERSION}" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml deleted file mode 100644 index e6dce7a8c664..000000000000 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ /dev/null @@ -1,167 +0,0 @@ ---- -name: "bitcoin-core-linux-22" -enable_cache: true -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -# Common dependencies. -- "autoconf" -- "automake" -- "binutils" -- "bison" -- "bsdmainutils" -- "ca-certificates" -- "curl" -- "faketime" -- "g++-8" -- "gcc-8" -- "git" -- "libtool" -- "patch" -- "pkg-config" -- "python3" -- "python3-pip" -# Cross compilation HOSTS: -# - arm-linux-gnueabihf -- "binutils-arm-linux-gnueabihf" -- "g++-8-arm-linux-gnueabihf" -# - aarch64-linux-gnu -- "binutils-aarch64-linux-gnu" -- "g++-8-aarch64-linux-gnu" -# - powerpc64-linux-gnu -- "binutils-powerpc64-linux-gnu" -- "g++-8-powerpc64-linux-gnu" -# - powerpc64le-linux-gnu -- "binutils-powerpc64le-linux-gnu" -- "g++-8-powerpc64le-linux-gnu" -# - riscv64-linux-gnu -- "binutils-riscv64-linux-gnu" -- "g++-8-riscv64-linux-gnu" -remotes: -- "url": "https://github.com/bitcoin/bitcoin.git" - "dir": "bitcoin" -files: [] -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu riscv64-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" - FAKETIME_HOST_PROGS="gcc g++" - FAKETIME_PROGS="date ar ranlib nm" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS_BASE="-static-libstdc++ -Wl,-O2" - - export TZ="UTC" - export BUILD_DIR="$PWD" - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - if which ${i}-${prog}-8 - then - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - if [ "${i:0:11}" = "powerpc64le" ]; then - echo "exec \"\$REAL\" -mcpu=power8 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} - elif [ "${i:0:9}" = "powerpc64" ]; then - echo "exec \"\$REAL\" -mcpu=970 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} - else - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - fi - chmod +x ${WRAP_DIR}/${i}-${prog} - fi - done - done - } - - pip3 install lief==0.11.5 - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd bitcoin - BASEPREFIX="${PWD}/depends" - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" CC=${i}-gcc-8 CXX=${i}-g++-8 - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Define DISTNAME variable. - # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME - source contrib/gitian-descriptors/assign_DISTNAME - - GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - - # Create the source tarball - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - - ORIGPATH="$PATH" - # Extract the git archive into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - if [ "${i}" = "powerpc64-linux-gnu" ]; then - # Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740 - # TODO: remove this when no longer needed - HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack" - else - HOST_LDFLAGS="${HOST_LDFLAGS_BASE}" - fi - mkdir -p distsrc-${i} - cd distsrc-${i} - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" CC=${i}-gcc-8 CXX=${i}-g++-8 - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make install DESTDIR=${INSTALLPATH} - cd installed - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - cp ../README.md ${DISTNAME}/ - find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz - cd ../../ - rm -rf distsrc-${i} - done diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml deleted file mode 100644 index addad0a5d27a..000000000000 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: "bitcoin-dmg-signer" -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "faketime" -- "xorriso" -- "python3-pip" -remotes: -- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" - "dir": "signature" -- "url": "https://github.com/achow101/signapple.git" - "dir": "signapple" - "commit": "b084cbbf44d5330448ffce0c7d118f75781b64bd" -files: -- "bitcoin-osx-unsigned.tar.gz" -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - mkdir -p ${WRAP_DIR} - export PATH="$PWD":$PATH - FAKETIME_PROGS="dmg xorrisofs" - - # Create global faketime wrappers - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - - # Install signapple - cd signapple - python3 -m pip install -U pip setuptools - python3 -m pip install . - export PATH="$HOME/.local/bin":$PATH - cd .. - - UNSIGNED_TARBALL=bitcoin-osx-unsigned.tar.gz - UNSIGNED_APP=dist/Bitcoin-Qt.app - SIGNED=bitcoin-osx-signed.dmg - - tar -xf ${UNSIGNED_TARBALL} - OSX_VOLNAME="$(cat osx_volname)" - ./detached-sig-apply.sh ${UNSIGNED_APP} signature/osx/dist - ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app - ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml deleted file mode 100644 index a39618adb78d..000000000000 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ /dev/null @@ -1,155 +0,0 @@ ---- -name: "bitcoin-core-osx-22" -enable_cache: true -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "ca-certificates" -- "curl" -- "g++" -- "git" -- "pkg-config" -- "autoconf" -- "librsvg2-bin" -- "libtiff-tools" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "cmake" -- "imagemagick" -- "libz-dev" -- "python3" -- "python3-pip" -- "python3-setuptools" -- "fonts-tuffy" -- "xorriso" -- "libtinfo5" -remotes: -- "url": "https://github.com/bitcoin/bitcoin.git" - "dir": "bitcoin" -files: -- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz" -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin18" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" - FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg xorrisofs" - - export TZ="UTC" - export BUILD_DIR="$PWD" - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - export ZERO_AR_DATE=1 - - # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - pip3 install lief==0.11.5 - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd bitcoin - BASEPREFIX="${PWD}/depends" - - mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz - - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Define DISTNAME variable. - # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME - source contrib/gitian-descriptors/assign_DISTNAME - - GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - - # Create the source tarball - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - - ORIGPATH="$PATH" - # Extract the git archive into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir -p distsrc-${i} - cd distsrc-${i} - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make install-strip DESTDIR=${INSTALLPATH} - - make osx_volname - make deploydir - mkdir -p unsigned-app-${i} - cp osx_volname unsigned-app-${i}/ - cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} - cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i} - mv dist unsigned-app-${i} - pushd unsigned-app-${i} - find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz - popd - - make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" - - cd installed - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME} | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - cd ../../ - done - - mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml deleted file mode 100644 index c13c24c3cc33..000000000000 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: "bitcoin-win-signer" -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "libssl-dev" -- "autoconf" -- "automake" -- "libtool" -- "pkg-config" -remotes: -- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" - "dir": "signature" -files: -- "osslsigncode-2.0.tar.gz" -- "bitcoin-win-unsigned.tar.gz" -script: | - set -e -o pipefail - - BUILD_DIR="$PWD" - SIGDIR=${BUILD_DIR}/signature/win - UNSIGNED_DIR=${BUILD_DIR}/unsigned - - echo "5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f osslsigncode-2.0.tar.gz" | sha256sum -c - - mkdir -p ${UNSIGNED_DIR} - tar -C ${UNSIGNED_DIR} -xf bitcoin-win-unsigned.tar.gz - - tar xf osslsigncode-2.0.tar.gz - cd osslsigncode-2.0 - - ./autogen.sh - ./configure --without-gsf --without-curl --disable-dependency-tracking - make - find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do - INFILE="$(basename "${i}")" - OUTFILE="${INFILE/-unsigned}" - ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem" - done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml deleted file mode 100644 index ffe228a032e6..000000000000 --- a/contrib/gitian-descriptors/gitian-win.yml +++ /dev/null @@ -1,157 +0,0 @@ ---- -name: "bitcoin-core-win-22" -enable_cache: true -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "curl" -- "g++" -- "git" -- "pkg-config" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "mingw-w64" -- "g++-mingw-w64" -- "nsis" -- "zip" -- "ca-certificates" -- "python3" -- "python3-pip" -remotes: -- "url": "https://github.com/bitcoin/bitcoin.git" - "dir": "bitcoin" -files: [] -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" - FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" - FAKETIME_PROGS="date makensis zip" - HOST_CFLAGS="-O2 -g -fno-ident" - HOST_CXXFLAGS="-O2 -g -fno-ident" - - export TZ="UTC" - export BUILD_DIR="$PWD" - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - function create_per-host_compiler_wrapper { - # -posix variant is required for c++11 threading. - for i in $HOSTS; do - for prog in gcc g++; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - pip3 install lief==0.11.5 - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_compiler_wrapper "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd bitcoin - BASEPREFIX="${PWD}/depends" - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_compiler_wrapper "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Define DISTNAME variable. - # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME - source contrib/gitian-descriptors/assign_DISTNAME - - GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - - # Create the source tarball - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - - ORIGPATH="$PATH" - # Extract the git archive into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir -p distsrc-${i} - cd distsrc-${i} - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make deploy BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - make install DESTDIR=${INSTALLPATH} - cd installed - mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - cp ../doc/README_windows.txt ${DISTNAME}/readme.txt - find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i//x86_64-w64-mingw32/win64}.zip - find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i//x86_64-w64-mingw32/win64}-debug.zip - cd ../../ - rm -rf distsrc-${i} - done - - cp -rf contrib/windeploy $BUILD_DIR - cd $BUILD_DIR/windeploy - mkdir unsigned - cp ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe unsigned/ - find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md index 86f91cc87bf0..bbd88e58f3dc 100644 --- a/contrib/guix/INSTALL.md +++ b/contrib/guix/INSTALL.md @@ -72,15 +72,15 @@ writing (July 2021). Guix is expected to be more widely packaged over time. For an up-to-date view on Guix's package status/version across distros, please see: https://repology.org/project/guix/versions -### Debian 11 (Bullseye)/Ubuntu 21.04 (Hirsute Hippo) +### Debian / Ubuntu Guix v1.2.0 is available as a distribution package starting in [Debian 11](https://packages.debian.org/bullseye/guix) and [Ubuntu -21.04](https://packages.ubuntu.com/hirsute/guix). +21.04](https://packages.ubuntu.com/search?keywords=guix). Note that if you intend on using Guix without using any substitutes (more -details [here][security-model]), v1.2.0 has a known problems when building -GnuTLS from source. Solutions and workarounds are documented +details [here][security-model]), v1.2.0 has a known problem when building GnuTLS +from source. Solutions and workarounds are documented [here](#gnutls-test-suite-fail-status-request-revoked). @@ -124,7 +124,7 @@ particular commit of Guix). Previous experience with using autotools-style build systems to build packages from source will be helpful. *hic sunt dracones.* I strongly urge you to at least skim through the entire section once before you -start issuing commands, as it will save you a lot of unncessary pain and +start issuing commands, as it will save you a lot of unnecessary pain and anguish. ### Installing common build tools @@ -165,7 +165,11 @@ packaged and installable without manually building and installing. For reference, the graphic below outlines Guix v1.3.0's dependency graph: -![boostrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) +![bootstrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) + +#### Consider /tmp on tmpfs + +If you use an NVME (SSD) drive, you may encounter [cryptic build errors](#coreutils-fail-teststail-2inotify-dir-recreate). Mounting a [tmpfs at /tmp](https://ubuntu.com/blog/data-driven-analysis-tmp-on-tmpfs) should prevent this and may improve performance as a bonus. #### Guile @@ -270,23 +274,11 @@ Note that these environment variables are used to check for packages during `./configure`, so they should be set as soon as possible should you want to use a prefix other than `/usr`. - - - - - - - - - - - - #### Building and installing source-built packages -***IMPORTANT**: A few dependencies have non-obvious quirks/erratas which are documented in the -sub-sections immediately below. Please read these sections before proceeding to -build and install these packages.* +***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are +documented in the sub-sections immediately below. Please read these sections +before proceeding to build and install these packages.* Although you should always refer to the README or INSTALL files for the most accurate information, most of these dependencies use autoconf-style build @@ -346,6 +338,8 @@ packages in Debian at the time of writing. |-----------------------|---------------------| | guile-gcrypt | libgcrypt-dev | | guile-git | libgit2-dev | +| guile-gnutls | (none) | +| guile-json | (none) | | guile-lzlib | liblz-dev | | guile-ssh | libssh-dev | | guile-sqlite3 | libsqlite3-dev | @@ -370,7 +364,7 @@ This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and Should you be in this situation, you need to build both `libgit2 v1.1.x` and `guile-git` from source. -Source: http://logs.guix.gnu.org/guix/2020-11-12.log#232527 +Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 ##### `{scheme,guile}-bytestructures` v1.0.8 and v1.0.9 are broken for Guile v2.2 @@ -396,8 +390,9 @@ cd guix ``` You will likely want to build the latest release, however, if the latest release -when you're reading this is still 1.2.0 then you may want to use 95aca29 instead -to avoid a problem in the GnuTLS test suite. +when you're reading this is still 1.3.0 then you may want to use 998eda30 instead +to avoid the issues described in [#25099]( +https://github.com/bitcoin/bitcoin/pull/25099). ``` git branch -a -l 'origin/version-*' # check for the latest release @@ -621,6 +616,8 @@ systemctl enable guix-daemon systemctl start guix-daemon ``` +Remember to set `--no-substitute` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. + ##### If you installed Guix via the Debian/Ubuntu distribution packages You will need to create a `guix-daemon-latest` service which points to the new @@ -729,6 +726,19 @@ $ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build failure output for the most accurate, up-to-date information. +### openssl-1.1.1l and openssl-1.1.1n + +OpenSSL includes tests that will fail once some certificate has expired. A workaround +is to change your system clock: + +```sh +sudo timedatectl set-ntp no +sudo date --set "28 may 2022 15:00:00" +sudo --login guix build --cores=1 /gnu/store/g9alz81w4q03ncm542487xd001s6akd4-openssl-1.1.1l.drv +sudo --login guix build --cores=1 /gnu/store/mw6ax0gk33gh082anrdrxp2flrbskxv6-openssl-1.1.1n.drv +sudo timedatectl set-ntp yes +``` + ### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem @@ -786,7 +796,7 @@ The inotify-dir-create test fails on "remote" filesystems such as overlayfs as non-remote. A relatively easy workaround to this is to make sure that a somewhat traditional -filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For +filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds), see [/tmp on tmpfs](#consider-tmp-on-tmpfs). For Docker users, this might mean [using a volume][docker/volumes], [binding mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) [mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. @@ -794,7 +804,7 @@ mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) Please see the following links for more details: - An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) -- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935) +- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935), [guix-issues#49985](https://issues.guix.gnu.org/49985#5) - A commit to skip this test in Guix has been merged into the core-updates branch: [savannah/guix@6ba1058](https://git.savannah.gnu.org/cgit/guix.git/commit/?id=6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) @@ -811,3 +821,39 @@ Please see the following links for more details: [docker/volumes]: https://docs.docker.com/storage/volumes/ [docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ [docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ + +# Purging/Uninstalling Guix + +In the extraordinarily rare case where you messed up your Guix installation in +an irreversible way, you may want to completely purge Guix from your system and +start over. + +1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt + purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). +2. Remove all build users and groups + + You may check for relevant users and groups using: + + ``` + getent passwd | grep guix + getent group | grep guix + ``` + + Then, you may remove users and groups using: + + ``` + sudo userdel + sudo groupdel + ``` + +3. Remove all possible Guix-related directories + - `/var/guix/` + - `/var/log/guix/` + - `/gnu/` + - `/etc/guix/` + - `/home/*/.config/guix/` + - `/home/*/.cache/guix/` + - `/home/*/.guix-profile/` + - `/root/.config/guix/` + - `/root/.cache/guix/` + - `/root/.guix-profile/` diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 4680368a6f87..c0feb486ff2a 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -11,7 +11,7 @@ We achieve bootstrappability by using Guix as a functional package manager. # Requirements -Conservatively, a x86_64 machine with: +Conservatively, you will need an x86_64 machine with: - 16GB of free disk space on the partition that /gnu/store will reside in - 8GB of free disk space **per platform triple** you're planning on building @@ -75,7 +75,7 @@ crucial differences: 1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` environment variable will have a sane default value of `x86_64-w64-mingw32 - x86_64-apple-darwin18` instead of all the platforms. + x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. 2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. * _**DETACHED_SIGS_REPO**_ @@ -87,7 +87,7 @@ crucial differences: An invocation with all default options would look like: ``` -env DETACHED_SIGS_REPO= ./contrib/guix-codesign +env DETACHED_SIGS_REPO= ./contrib/guix/guix-codesign ``` ## Cleaning intermediate work directories @@ -159,7 +159,7 @@ which case you can override the default list by setting the space-separated `HOSTS` environment variable: ```sh -env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin18' ./contrib/guix/guix-build +env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build ``` See the [recognized environment variables][env-vars-list] section for more @@ -224,7 +224,7 @@ details. _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu - x86\_64-w64-mingw32 x86\_64-apple-darwin18")_ + x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_ * _**SOURCES_PATH**_ @@ -249,7 +249,7 @@ details. Set the path where _extracted_ SDKs can be found. This is passed through to the depends tree. Note that this is should be set to the _parent_ directory of the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of - `$HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers`). + `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`). The path that this environment variable points to **must be a directory**, and **NOT a symlink to a directory**. @@ -382,7 +382,7 @@ https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag is supplied. This default list of substitute servers is overridable both on a `guix-daemon` level and when you invoke `guix` commands. See examples below for the various ways of adding dongcarl's substitute server after having [authorized -his signing key](#authorize-the-signing-keys). +his signing key](#step-1-authorize-the-signing-keys). Change the **default list** of substitute servers by starting `guix-daemon` with the `--substitute-urls` option (you will likely need to edit your init script): @@ -430,56 +430,6 @@ used. If you start `guix-daemon` using an init script, you can edit said script to supply this flag. - -# Purging/Uninstalling Guix - -In the extraordinarily rare case where you messed up your Guix installation in -an irreversible way, you may want to completely purge Guix from your system and -start over. - -1. Uninstall Guix itself according to the way you installed it. (e.g. `sudo apt - purge guix` for Ubuntu packaging, `sudo make uninstall` for - built-from-source). -2. Remove all build users and groups - - You may check for relevant users and groups using: - - ``` - getent passwd | grep guix - getent group | grep guix - ``` - - Then, you may remove users and groups using: - - ``` - sudo userdel - sudo groupdel - ``` - -3. Remove all possible Guix-related directories - - `/var/guix/` - - `/var/log/guix/` - - `/gnu/` - - `/etc/guix/` - - `/home/*/.config/guix/` - - `/home/*/.cache/guix/` - - `/home/*/.guix-profile/` - - `/root/.config/guix/` - - `/root/.cache/guix/` - - `/root/.guix-profile/` - -[b17e]: http://bootstrappable.org/ +[b17e]: https://bootstrappable.org/ [r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ - -[guix/install.sh]: https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh -[guix/bin-install]: https://www.gnu.org/software/guix/manual/en/html_node/Binary-Installation.html -[guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html -[guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html -[guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html -[guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html - -[debian/guix-bullseye]: https://packages.debian.org/bullseye/guix -[ubuntu/guix-hirsute]: https://packages.ubuntu.com/hirsute/guix -[fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix - [env-vars-list]: #recognized-environment-variables diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 51d589c1dee3..b0ef28dc3f92 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -19,8 +19,16 @@ source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" ################ check_tools cat env basename mkdir diff sort + if [ -z "$NO_SIGN" ]; then - check_tools gpg + # make it possible to override the gpg binary + GPG=${GPG:-gpg} + + # $GPG can contain extra arguments passed to the binary + # so let's check only the existence of arg[0] + # shellcheck disable=SC2206 + GPG_ARRAY=($GPG) + check_tools "${GPG_ARRAY[0]}" fi ################ @@ -90,7 +98,7 @@ if [ -z "${signer_name}" ]; then signer_name="$gpg_key_name" fi -if [ -z "$NO_SIGN" ] && ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then +if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" exit 1 fi @@ -159,23 +167,21 @@ Hint: You may wish to remove the existing attestations and their signatures by EOF } -# Given a document with unix line endings (just ) in stdin, make all lines -# end in and make sure there's no trailing at the end of the file. -# -# This is necessary as cleartext signatures are calculated on text after their -# line endings are canonicalized. +echo "Attesting to build outputs for version: '${VERSION}'" +echo "" + +# Given a SHA256SUMS file as stdin that has lines like: +# 0ba536819b221a91d3d42e978be016aac918f40984754d74058aa0c921cd3ea6 a/b/d/c/d/s/bitcoin-22.0rc2-riscv64-linux-gnu.tar.gz +# ... # -# For more information: -# 1. https://security.stackexchange.com/a/104261 -# 2. https://datatracker.ietf.org/doc/html/rfc4880#section-7.1 +# Replace each line's file name with its basename: +# 0ba536819b221a91d3d42e978be016aac918f40984754d74058aa0c921cd3ea6 bitcoin-22.0rc2-riscv64-linux-gnu.tar.gz +# ... # -rfc4880_normalize_document() { - sed 's/$/\r/' | head -c -2 +basenameify_SHA256SUMS() { + sed -E 's@(^[[:xdigit:]]{64}[[:space:]]+).+/([^/]+$)@\1\2@' } -echo "Attesting to build outputs for version: '${VERSION}'" -echo "" - outsigdir="$GUIX_SIGS_REPO/$VERSION/$signer_name" mkdir -p "$outsigdir" ( @@ -188,7 +194,7 @@ mkdir -p "$outsigdir" cat "${noncodesigned_fragments[@]}" \ | sort -u \ | sort -k2 \ - | rfc4880_normalize_document \ + | basenameify_SHA256SUMS \ > "$temp_noncodesigned" if [ -e noncodesigned.SHA256SUMS ]; then # The SHA256SUMS already exists, make sure it's exactly what we @@ -207,8 +213,8 @@ mkdir -p "$outsigdir" exit 1 fi - temp_codesigned="$(mktemp)" - trap 'rm -rf -- "$temp_codesigned"' EXIT + temp_all="$(mktemp)" + trap 'rm -rf -- "$temp_all"' EXIT if (( ${#codesigned_fragments[@]} )); then # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is @@ -216,20 +222,19 @@ mkdir -p "$outsigdir" cat "${sha256sum_fragments[@]}" \ | sort -u \ | sort -k2 \ - | sed 's/$/\r/' \ - | rfc4880_normalize_document \ - > "$temp_codesigned" - if [ -e codesigned.SHA256SUMS ]; then + | basenameify_SHA256SUMS \ + > "$temp_all" + if [ -e all.SHA256SUMS ]; then # The SHA256SUMS already exists, make sure it's exactly what we # expect, error out if not - if diff -u all.SHA256SUMS "$temp_codesigned"; then + if diff -u all.SHA256SUMS "$temp_all"; then echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." else shasum_already_exists all.SHA256SUMS exit 1 fi else - mv "$temp_codesigned" codesigned.SHA256SUMS + mv "$temp_all" all.SHA256SUMS fi else # It is fine to have the codesigned outputs be missing (perhaps the @@ -242,11 +247,11 @@ mkdir -p "$outsigdir" echo "Signing SHA256SUMS to produce SHA256SUMS.asc" for i in *.SHA256SUMS; do if [ ! -e "$i".asc ]; then - gpg --detach-sign \ - --digest-algo sha256 \ - --local-user "$gpg_key_name" \ - --armor \ - --output "$i".asc "$i" + ${GPG} --detach-sign \ + --digest-algo sha256 \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" else echo "Signature already there" fi diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index f6da8435e99e..74b24b961207 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -76,7 +76,7 @@ mkdir -p "$VERSION_BASE" # Default to building for all supported HOSTs (overridable by environment) export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu x86_64-w64-mingw32 - x86_64-apple-darwin18}" + x86_64-apple-darwin arm64-apple-darwin}" # Usage: distsrc_for_host HOST # @@ -121,7 +121,7 @@ else fi ################ -# When building for darwin, the macOS SDK should exists +# When building for darwin, the macOS SDK should exist ################ for host in $HOSTS; do @@ -130,8 +130,9 @@ for host in $HOSTS; do OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" if [ -e "$OSX_SDK" ]; then echo "Found macOS SDK at '${OSX_SDK}', using..." + break else - echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." exit 1 fi ;; @@ -190,9 +191,9 @@ fi # Services database must have basic entries ################ -if ! getent services http https ftp; then +if ! getent services http https ftp > /dev/null 2>&1; then cat << EOF -ERR: Your system's C library can not find service database entries for at least +ERR: Your system's C library cannot find service database entries for at least one of the following services: http, https, ftp. Hint: Most likely, /etc/services does not exist yet (common for docker images @@ -232,22 +233,7 @@ host_to_commonname() { } # Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" - -# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility -# across time. -time-machine() { - # shellcheck disable=SC2086 - guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" -} - +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" # Precious directories are those which should not be cleaned between successive # guix builds diff --git a/contrib/guix/guix-clean b/contrib/guix/guix-clean index 9fa17191e808..9af0a793cff7 100755 --- a/contrib/guix/guix-clean +++ b/contrib/guix/guix-clean @@ -34,7 +34,7 @@ check_tools cat mkdir make git guix # under_dir() { local path_residue - path_residue="${2##${1}}" + path_residue="${2##"${1}"}" if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then return 1 else diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 11610a92e118..3279d431aaf3 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -91,7 +91,7 @@ fi ################ # Default to building for all supported HOSTs (overridable by environment) -export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin18}" +export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" # Usage: distsrc_for_host HOST # @@ -152,10 +152,10 @@ outdir_for_host() { unsigned_tarball_for_host() { case "$1" in *mingw*) - echo "$(outdir_for_host "$1")/${DISTNAME}-win-unsigned.tar.gz" + echo "$(outdir_for_host "$1")/${DISTNAME}-win64-unsigned.tar.gz" ;; *darwin*) - echo "$(outdir_for_host "$1")/${DISTNAME}-osx-unsigned.tar.gz" + echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-unsigned.tar.gz" ;; *) exit 1 @@ -220,21 +220,7 @@ fi JOBS="${JOBS:-$(nproc)}" # Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" - -# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility -# across time. -time-machine() { - # shellcheck disable=SC2086 - guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" -} +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" # Make sure an output directory exists for our builds OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" diff --git a/contrib/guix/guix-verify b/contrib/guix/guix-verify index a6e2c4065ecc..02ae022741ba 100755 --- a/contrib/guix/guix-verify +++ b/contrib/guix/guix-verify @@ -28,7 +28,11 @@ cmd_usage() { cat < ./contrib/guix/guix-verify + env GUIX_SIGS_REPO= [ SIGNER= ] ./contrib/guix/guix-verify + +Example overriding signer's manifest to use as base + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify EOF } @@ -73,11 +77,13 @@ verify() { echo "" echo "Hint: Either the signature is invalid or the public key is missing" echo "" + failure=1 elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then echo "ERR: The SHA256SUMS attestation in these two directories differ:" echo " '${compare_manifest}'" echo " '${current_manifest}'" echo "" + failure=1 else echo "Verified: '${current_manifest}'" echo "" @@ -92,6 +98,17 @@ echo "--------------------" echo "" if (( ${#all_noncodesigned[@]} )); then compare_noncodesigned="${all_noncodesigned[0]}" + if [[ -n "$SIGNER" ]]; then + signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" + if [[ -f "$signer_noncodesigned" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_noncodesigned="$signer_noncodesigned" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi for current_manifest in "${all_noncodesigned[@]}"; do verify "$compare_noncodesigned" "$current_manifest" @@ -112,6 +129,17 @@ echo "--------------------" echo "" if (( ${#all_all[@]} )); then compare_all="${all_all[0]}" + if [[ -n "$SIGNER" ]]; then + signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" + if [[ -f "$signer_all" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_all="$signer_all" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi for current_manifest in "${all_all[@]}"; do verify "$compare_all" "$current_manifest" @@ -140,3 +168,7 @@ if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then echo "" exit 1 fi + +if [ -n "$failure" ]; then + exit 1 +fi diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index bc3391e0896e..f26b6795d16e 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Copyright (c) 2019-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C set -e -o pipefail export TZ=UTC @@ -66,29 +69,12 @@ unset CPLUS_INCLUDE_PATH unset OBJC_INCLUDE_PATH unset OBJCPLUS_INCLUDE_PATH -export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64:${NATIVE_GCC_STATIC}/lib:${NATIVE_GCC_STATIC}/lib64" +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" export C_INCLUDE_PATH="${NATIVE_GCC}/include" export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" -prepend_to_search_env_var() { - export "${1}=${2}${!1:+:}${!1}" -} - -case "$HOST" in - *darwin*) - # When targeting darwin, zlib is required by native_libdmg-hfsplus. - zlib_store_path=$(store_path "zlib") - zlib_static_store_path=$(store_path "zlib" static) - - prepend_to_search_env_var LIBRARY_PATH "${zlib_static_store_path}/lib:${zlib_store_path}/lib" - prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include" - prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include" - prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include" - prepend_to_search_env_var OBJCPLUS_INCLUDE_PATH "${zlib_store_path}/include" -esac - # Set environment variables to point the CROSS toolchain to the right # includes/libs for $HOST case "$HOST" in @@ -106,7 +92,7 @@ case "$HOST" in # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" ;; *darwin*) # The CROSS toolchain for darwin uses the SDK and ignores environment variables. @@ -123,7 +109,7 @@ case "$HOST" in export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" ;; *) exit 1 ;; @@ -147,7 +133,7 @@ case "$HOST" in # # After the native packages in depends are built, the ld wrapper should # no longer affect our build, as clang would instead reach for - # x86_64-apple-darwin18-ld from cctools + # x86_64-apple-darwin-ld from cctools ;; *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;; esac @@ -164,13 +150,12 @@ case "$HOST" in *linux*) glibc_dynamic_linker=$( case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - powerpc64-linux-gnu) echo /lib/ld64.so.1;; - powerpc64le-linux-gnu) echo /lib/ld64.so.2;; + powerpc64-linux-gnu) echo /lib64/ld64.so.1;; + powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; *) exit 1 ;; esac ) @@ -201,20 +186,12 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ - i686_linux_CC=i686-linux-gnu-gcc \ - i686_linux_CXX=i686-linux-gnu-g++ \ - i686_linux_AR=i686-linux-gnu-ar \ - i686_linux_RANLIB=i686-linux-gnu-ranlib \ - i686_linux_NM=i686-linux-gnu-nm \ - i686_linux_STRIP=i686-linux-gnu-strip \ x86_64_linux_CC=x86_64-linux-gnu-gcc \ x86_64_linux_CXX=x86_64-linux-gnu-g++ \ x86_64_linux_AR=x86_64-linux-gnu-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ - qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ FORCE_USE_SYSTEM_CLANG=1 @@ -227,7 +204,6 @@ GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" # Create the source tarball if not already there if [ ! -e "$GIT_ARCHIVE" ]; then mkdir -p "$(dirname "$GIT_ARCHIVE")" - touch "${DIST_ARCHIVE_BASE}"/SKIPATTEST.TAG git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD fi @@ -239,12 +215,10 @@ mkdir -p "$OUTDIR" # CONFIGFLAGS CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" -case "$HOST" in - *linux*) CONFIGFLAGS+=" --disable-threadlocal" ;; -esac # CFLAGS HOST_CFLAGS="-O2 -g" +HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) case "$HOST" in *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; *mingw*) HOST_CFLAGS+=" -fno-ident" ;; @@ -264,17 +238,13 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac -# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.17, by +# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by # avoiding a PowerPC64 optimisation available in glibc 2.22 and later. # https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html case "$HOST" in *powerpc64*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,--no-tls-get-addr-optimize" ;; esac -case "$HOST" in - powerpc64-linux-*|riscv64-linux-*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,-z,noexecstack" ;; -esac - # Make $HOST-specific native binaries from depends available in $PATH export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" mkdir -p "$DISTSRC" @@ -298,7 +268,7 @@ mkdir -p "$DISTSRC" ${HOST_CXXFLAGS:+CXXFLAGS="${HOST_CXXFLAGS}"} \ ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} - sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool + sed -i.old 's/-lstdc++ //g' config.status libtool # Build Bitcoin Core make --jobs="$JOBS" ${V:+V=1} @@ -341,18 +311,17 @@ mkdir -p "$DISTSRC" mkdir -p "unsigned-app-${HOST}" cp --target-directory="unsigned-app-${HOST}" \ osx_volname \ - contrib/macdeploy/detached-sig-{apply,create}.sh \ - "${BASEPREFIX}/${HOST}"/native/bin/dmg + contrib/macdeploy/detached-sig-create.sh mv --target-directory="unsigned-app-${HOST}" dist ( cd "unsigned-app-${HOST}" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) ) - make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" + make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.dmg" ;; esac ( @@ -378,7 +347,7 @@ mkdir -p "$DISTSRC" { find "${DISTNAME}/bin" -type f -executable -print0 find "${DISTNAME}/lib" -type f -print0 - } | xargs -0 -n1 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg ;; esac @@ -391,6 +360,12 @@ mkdir -p "$DISTSRC" ;; esac + # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh + # has not been run before buildling, this file will be a stub + cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" + + cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/" + # Finally, deterministically produce {non-,}debug binary tarballs ready # for release case "$HOST" in @@ -424,8 +399,8 @@ mkdir -p "$DISTSRC" find "${DISTNAME}" -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) ;; esac ) # $DISTSRC/installed @@ -440,8 +415,8 @@ mkdir -p "$DISTSRC" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" && exit 1 ) ) ;; esac diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index f484ac5774c7..9a5d3a1ce542 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C set -e -o pipefail export TZ=UTC @@ -81,14 +84,11 @@ mkdir -p "$DISTSRC" # Apply detached codesignatures to dist/ (in-place) signapple apply dist/Bitcoin-Qt.app codesignatures/osx/dist - # Make an uncompressed DMG from dist/ + # Make a DMG from dist/ xorrisofs -D -l -V "$(< osx_volname)" -no-pad -r -dir-mode 0755 \ - -o uncompressed.dmg \ + -o "${OUTDIR}/${DISTNAME}-${HOST}.dmg" \ dist \ -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH" - - # Compress uncompressed.dmg and output to OUTDIR - ./dmg dmg uncompressed.dmg "${OUTDIR}/${DISTNAME}-osx-signed.dmg" ;; *) exit 1 diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 9705607119fb..3eb8fc02dae6 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -2,10 +2,10 @@ export LC_ALL=C set -e -o pipefail -# shellcheck source=../../shell/realpath.bash +# shellcheck source=contrib/shell/realpath.bash source contrib/shell/realpath.bash -# shellcheck source=../../shell/git-utils.bash +# shellcheck source=contrib/shell/git-utils.bash source contrib/shell/git-utils.bash ################ @@ -45,11 +45,27 @@ EOF exit 1 fi +################ +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ + --commit=998eda3067c7d21e0d9bb3310d2f5a14b8f1c681 \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + + ################ # Set common variables ################ -VERSION="${VERSION:-$(git_head_version)}" +VERSION="${FORCE_VERSION:-$(git_head_version)}" DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" version_base_prefix="${PWD}/guix-build-" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 580500605370..8e5c89cc5e0a 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -16,21 +16,18 @@ (gnu packages gawk) (gnu packages gcc) (gnu packages gnome) - (gnu packages image) - (gnu packages imagemagick) (gnu packages installers) (gnu packages linux) (gnu packages llvm) (gnu packages mingw) (gnu packages moreutils) - (gnu packages perl) (gnu packages pkg-config) (gnu packages python) + (gnu packages python-crypto) (gnu packages python-web) (gnu packages shells) (gnu packages tls) (gnu packages version-control) - (guix build-system font) (guix build-system gnu) (guix build-system python) (guix build-system trivial) @@ -80,9 +77,10 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (("-rpath=") "-rpath-link=")) #t)))))))) -(define (make-binutils-with-mingw-w64-disable-flags xbinutils) - (package-with-extra-patches xbinutils - (search-our-patches "binutils-mingw-w64-disable-flags.patch"))) +(define building-on (string-append (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) + +(define (explicit-cross-configure package) + (package-with-extra-configure-variable package "--build" building-on)) (define (make-cross-toolchain target base-gcc-for-libc @@ -93,9 +91,9 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (let* ((xbinutils (cross-binutils target)) ;; 1. Build a cross-compiling gcc without targeting any libc, derived ;; from BASE-GCC-FOR-LIBC - (xgcc-sans-libc (cross-gcc target - #:xgcc base-gcc-for-libc - #:xbinutils xbinutils)) + (xgcc-sans-libc (explicit-cross-configure (cross-gcc target + #:xgcc base-gcc-for-libc + #:xbinutils xbinutils))) ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived ;; from BASE-KERNEL-HEADERS (xkernel (cross-kernel-headers target @@ -104,17 +102,17 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" xbinutils)) ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, ;; derived from BASE-LIBC - (xlibc (cross-libc target - base-libc - xgcc-sans-libc - xbinutils - xkernel)) + (xlibc (explicit-cross-configure (cross-libc target + base-libc + xgcc-sans-libc + xbinutils + xkernel))) ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from ;; BASE-GCC - (xgcc (cross-gcc target - #:xgcc base-gcc - #:xbinutils xbinutils - #:libc xlibc))) + (xgcc (explicit-cross-configure (cross-gcc target + #:xgcc base-gcc + #:xbinutils xbinutils + #:libc xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and ;; XGCC (package @@ -135,30 +133,22 @@ chain for " target " development.")) (home-page (package-home-page xgcc)) (license (package-license xgcc))))) -(define base-gcc - (package-with-extra-patches gcc-8 - (search-our-patches "gcc-8-sort-libtool-find-output.patch"))) - -;; Building glibc with stack smashing protector first landed in glibc 2.25, use -;; this function to disable for older glibcs -;; -;; From glibc 2.25 changelog: -;; -;; * Most of glibc can now be built with the stack smashing protector enabled. -;; It is recommended to build glibc with --enable-stack-protector=strong. -;; Implemented by Nick Alcock (Oracle). -(define (make-glibc-without-ssp xglibc) - (package-with-extra-configure-variable - (package-with-extra-configure-variable - xglibc "libc_cv_ssp" "no") - "libc_cv_ssp_strong" "no")) +(define base-gcc gcc-10) +(define base-linux-kernel-headers linux-libre-headers-5.15) + +;; https://gcc.gnu.org/install/configure.html +(define (hardened-gcc gcc) + (package-with-extra-configure-variable ( + package-with-extra-configure-variable gcc + "--enable-default-ssp" "yes") + "--enable-default-pie" "yes")) (define* (make-bitcoin-cross-toolchain target #:key - (base-gcc-for-libc gcc-7) - (base-kernel-headers linux-libre-headers-4.9) - (base-libc (make-glibc-without-ssp glibc-2.24)) - (base-gcc (make-gcc-rpath-link base-gcc))) + (base-gcc-for-libc base-gcc) + (base-kernel-headers base-linux-kernel-headers) + (base-libc (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.24))) + (base-gcc (make-gcc-rpath-link (hardened-gcc base-gcc)))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target @@ -168,15 +158,23 @@ desirable for building Bitcoin Core release binaries." base-gcc)) (define (make-gcc-with-pthreads gcc) - (package-with-extra-configure-variable gcc "--enable-threads" "posix")) + (package-with-extra-configure-variable + (package-with-extra-patches gcc + (search-our-patches "gcc-10-remap-guix-store.patch")) + "--enable-threads" "posix")) + +(define (make-mingw-w64-cross-gcc cross-gcc) + (package-with-extra-patches cross-gcc + (search-our-patches "vmov-alignment.patch" + "gcc-broken-longjmp.patch"))) (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (make-binutils-with-mingw-w64-disable-flags (cross-binutils target))) + (let* ((xbinutils (cross-binutils target)) (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target - #:xgcc (make-ssp-fixed-gcc base-gcc) + #:xgcc (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc)) #:xbinutils xbinutils #:libc pthreads-xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and @@ -198,33 +196,19 @@ chain for " target " development.")) (home-page (package-home-page pthreads-xgcc)) (license (package-license pthreads-xgcc))))) -(define (make-nsis-with-sde-support base-nsis) +(define (make-nsis-for-gcc-10 base-nsis) (package-with-extra-patches base-nsis - (search-our-patches "nsis-SConstruct-sde-support.patch"))) + (search-our-patches "nsis-gcc-10-memmove.patch" + "nsis-disable-installer-reloc.patch"))) -(define-public font-tuffy - (package - (name "font-tuffy") - (version "20120614") - (source - (origin - (method url-fetch) - (uri (string-append "http://tulrich.com/fonts/tuffy-" version ".tar.gz")) - (file-name (string-append name "-" version ".tar.gz")) - (sha256 - (base32 - "02vf72bgrp30vrbfhxjw82s115z27dwfgnmmzfb0n9wfhxxfpyf6")))) - (build-system font-build-system) - (home-page "http://tulrich.com/fonts/") - (synopsis "The Tuffy Truetype Font Family") - (description - "Thatcher Ulrich's first outline font design. He started with the goal of producing a neutral, readable sans-serif text font. There are lots of \"expressive\" fonts out there, but he wanted to start with something very plain and clean, something he might want to actually use. ") - (license license:public-domain))) +(define (fix-ppc64-nx-default lief) + (package-with-extra-patches lief + (search-our-patches "lief-fix-ppc64-nx-default.patch"))) (define-public lief (package (name "python-lief") - (version "0.11.5") + (version "0.12.1") (source (origin (method git-fetch) @@ -234,8 +218,15 @@ chain for " target " development.")) (file-name (git-file-name name version)) (sha256 (base32 - "0qahjfg1n0x76ps2mbyljvws1l3qhkqvmxqbahps4qgywl2hbdkj")))) + "1xzbh3bxy4rw1yamnx68da1v5s56ay4g081cyamv67256g0qy2i1")))) (build-system python-build-system) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'parallel-jobs + ;; build with multiple cores + (lambda _ + (substitute* "setup.py" (("self.parallel if self.parallel else 1") (number->string (parallel-job-count))))))))) (native-inputs `(("cmake" ,cmake))) (home-page "https://github.com/lief-project/LIEF") @@ -276,36 +267,8 @@ signing and timestamping. But osslsigncode is based on OpenSSL and cURL, and thus should be able to compile on most platforms where these exist.") (license license:gpl3+))) ; license is with openssl exception -(define-public python-asn1crypto - (package - (name "python-asn1crypto") - (version "1.4.0") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/wbond/asn1crypto") - (commit version))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "19abibn6jw20mzi1ln4n9jjvpdka8ygm4m439hplyrdfqbvgm01r")))) - (build-system python-build-system) - (arguments - '(#:phases - (modify-phases %standard-phases - (replace 'check - (lambda _ - (invoke "python" "run.py" "tests")))))) - (home-page "https://github.com/wbond/asn1crypto") - (synopsis "ASN.1 parser and serializer in Python") - (description "asn1crypto is an ASN.1 parser and serializer with definitions -for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, -PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") - (license license:expat))) - (define-public python-elfesteem - (let ((commit "87bbd79ab7e361004c98cc8601d4e5f029fd8bd5")) + (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) (package (name "python-elfesteem") (version (git-version "0.1" "1" commit)) @@ -318,7 +281,7 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (file-name (git-file-name name commit)) (sha256 (base32 - "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")))) + "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) (build-system python-build-system) ;; There are no tests, but attempting to run python setup.py test leads to ;; PYTHONPATH problems, just disable the test @@ -390,6 +353,8 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (define-public python-oscryptotests (package (inherit python-oscrypto) (name "python-oscryptotests") + (propagated-inputs + `(("python-oscrypto" ,python-oscrypto))) (arguments `(#:tests? #f #:phases @@ -400,7 +365,7 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") #t))))))) (define-public python-certvalidator - (let ((commit "e5bdb4bfcaa09fa0af355eb8867d00dfeecba08c")) + (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) (package (name "python-certvalidator") (version (git-version "0.1" "1" commit)) @@ -413,7 +378,7 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (file-name (git-file-name name commit)) (sha256 (base32 - "18pvxkvpkfkzgvfylv0kx65pmxfcv1hpsg03cip93krfvrrl4c75")))) + "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) (build-system python-build-system) (propagated-inputs `(("python-asn1crypto" ,python-asn1crypto) @@ -449,6 +414,11 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (string-append indent "@unittest.skip(\"Disabled by Guix\")\n" line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_soft" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) #t)) (replace 'check (lambda _ @@ -461,16 +431,6 @@ certificates or paths. Supports various options, including: validation at a specific moment in time, whitelisting and revocation checks.") (license license:expat)))) -(define-public python-requests-2.25.1 - (package (inherit python-requests) - (version "2.25.1") - (source (origin - (method url-fetch) - (uri (pypi-uri "requests" version)) - (sha256 - (base32 - "015qflyqsgsz09gnar69s6ga74ivq5kch69s4qxz3904m7a3v5r7")))))) - (define-public python-altgraph (package (name "python-altgraph") @@ -543,7 +503,7 @@ and endian independent.") (license license:expat))) (define-public python-signapple - (let ((commit "b084cbbf44d5330448ffce0c7d118f75781b64bd")) + (let ((commit "8a945a2e7583be2665cf3a6a89d665b70ecd1ab6")) (package (name "python-signapple") (version (git-version "0.1" "1" commit)) @@ -556,16 +516,15 @@ and endian independent.") (file-name (git-file-name name commit)) (sha256 (base32 - "0k7inccl2mzac3wq4asbr0kl8s4cghm8982z54kfascqg45shv01")))) + "0fr1hangvfyiwflca6jg5g8zvg3jc9qr7vd2c12ff89pznf38dlg")))) (build-system python-build-system) (propagated-inputs `(("python-asn1crypto" ,python-asn1crypto) ("python-oscrypto" ,python-oscrypto) ("python-certvalidator" ,python-certvalidator) ("python-elfesteem" ,python-elfesteem) - ("python-requests" ,python-requests-2.25.1) - ("python-macholib" ,python-macholib) - ("libcrypto" ,openssl))) + ("python-requests" ,python-requests) + ("python-macholib" ,python-macholib))) ;; There are no tests, but attempting to run python setup.py test leads to ;; problems, just disable the test (arguments '(#:tests? #f)) @@ -575,9 +534,18 @@ and endian independent.") inspecting signatures in Mach-O binaries.") (license license:expat)))) +(define (make-glibc-without-werror glibc) + (package-with-extra-configure-variable glibc "enable_werror" "no")) + +(define (make-glibc-with-stack-protector glibc) + (package-with-extra-configure-variable glibc "--enable-stack-protector" "all")) + +(define (make-glibc-with-bind-now glibc) + (package-with-extra-configure-variable glibc "--enable-bind-now" "yes")) + (define-public glibc-2.24 (package - (inherit glibc) + (inherit glibc-2.31) (version "2.24") (source (origin (method git-fetch) @@ -591,11 +559,27 @@ inspecting signatures in Mach-O binaries.") (patches (search-our-patches "glibc-ldd-x86_64.patch" "glibc-versioned-locpath.patch" "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" - "glibc-2.24-no-build-time-cxx-header-run.patch")))))) + "glibc-2.24-no-build-time-cxx-header-run.patch" + "glibc-2.24-fcommon.patch" + "glibc-2.24-guix-prefix.patch")))))) -(define glibc-2.27/bitcoin-patched - (package-with-extra-patches glibc-2.27 - (search-our-patches "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch"))) +(define-public glibc-2.27/bitcoin-patched + (package + (inherit glibc-2.31) + (version "2.27") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit "23158b08a0908f381459f273a984c6fd328363cb"))) + (file-name (git-file-name "glibc" "23158b08a0908f381459f273a984c6fd328363cb")) + (sha256 + (base32 + "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca")) + (patches (search-our-patches "glibc-ldd-x86_64.patch" + "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" + "glibc-2.27-dont-redefine-nss-database.patch" + "glibc-2.27-guix-prefix.patch")))))) (packages->manifest (append @@ -619,39 +603,36 @@ inspecting signatures in Mach-O binaries.") bzip2 gzip xz - zlib - (list zlib "static") ;; Build tools gnu-make - libtool - autoconf + libtool-2.4.7 + autoconf-2.71 automake pkg-config bison + ;; Native GCC 10 toolchain + gcc-toolchain-10 + (list gcc-toolchain-10 "static") ;; Scripting - perl python-3 ;; Git - git + git-minimal ;; Tests - lief - ;; Native gcc 7 toolchain - gcc-toolchain-7 - (list gcc-toolchain-7 "static")) + (fix-ppc64-nx-default lief)) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") - (make-nsis-with-sde-support nsis-x86_64) + (make-nsis-for-gcc-10 nsis-x86_64) osslsigncode)) ((string-contains target "-linux-") (list (cond ((string-contains target "riscv64-") (make-bitcoin-cross-toolchain target - #:base-libc glibc-2.27/bitcoin-patched - #:base-kernel-headers linux-libre-headers-4.19)) + #:base-libc (make-glibc-with-stack-protector + (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.27/bitcoin-patched))))) (else (make-bitcoin-cross-toolchain target))))) ((string-contains target "darwin") - (list clang-toolchain-10 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso python-signapple)) + (list clang-toolchain-10 binutils cmake xorriso python-signapple)) (else '()))))) diff --git a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch deleted file mode 100644 index 8f88eb9dfd59..000000000000 --- a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch +++ /dev/null @@ -1,171 +0,0 @@ -Description: Add disable opposites to the security-related flags -Author: Stephen Kitt - -This patch adds "no-" variants to disable the various security flags: -"no-dynamicbase", "no-nxcompat", "no-high-entropy-va", "disable-reloc-section". - ---- a/ld/emultempl/pe.em -+++ b/ld/emultempl/pe.em -@@ -259,9 +261,11 @@ - (OPTION_ENABLE_LONG_SECTION_NAMES + 1) - /* DLLCharacteristics flags. */ - #define OPTION_DYNAMIC_BASE (OPTION_DISABLE_LONG_SECTION_NAMES + 1) --#define OPTION_FORCE_INTEGRITY (OPTION_DYNAMIC_BASE + 1) -+#define OPTION_NO_DYNAMIC_BASE (OPTION_DYNAMIC_BASE + 1) -+#define OPTION_FORCE_INTEGRITY (OPTION_NO_DYNAMIC_BASE + 1) - #define OPTION_NX_COMPAT (OPTION_FORCE_INTEGRITY + 1) --#define OPTION_NO_ISOLATION (OPTION_NX_COMPAT + 1) -+#define OPTION_NO_NX_COMPAT (OPTION_NX_COMPAT + 1) -+#define OPTION_NO_ISOLATION (OPTION_NO_NX_COMPAT + 1) - #define OPTION_NO_SEH (OPTION_NO_ISOLATION + 1) - #define OPTION_NO_BIND (OPTION_NO_SEH + 1) - #define OPTION_WDM_DRIVER (OPTION_NO_BIND + 1) -@@ -271,6 +275,7 @@ - #define OPTION_NO_INSERT_TIMESTAMP (OPTION_INSERT_TIMESTAMP + 1) - #define OPTION_BUILD_ID (OPTION_NO_INSERT_TIMESTAMP + 1) - #define OPTION_ENABLE_RELOC_SECTION (OPTION_BUILD_ID + 1) -+#define OPTION_DISABLE_RELOC_SECTION (OPTION_ENABLE_RELOC_SECTION + 1) - - static void - gld${EMULATION_NAME}_add_options -@@ -342,8 +347,10 @@ - {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, - {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, - {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, -+ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, - {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, - {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, -+ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, - {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, - {"no-seh", no_argument, NULL, OPTION_NO_SEH}, - {"no-bind", no_argument, NULL, OPTION_NO_BIND}, -@@ -351,6 +358,7 @@ - {"tsaware", no_argument, NULL, OPTION_TERMINAL_SERVER_AWARE}, - {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, - {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, -+ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, - {NULL, no_argument, NULL, 0} - }; - -@@ -485,9 +494,12 @@ - in object files\n")); - fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ - address space layout randomization (ASLR)\n")); -+ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); - fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); -+ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); - fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); - fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); -+ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); - fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); - fprintf (file, _(" --no-seh Image does not use SEH. No SE handler may\n\ - be called in this image\n")); -@@ -862,12 +874,21 @@ - case OPTION_ENABLE_RELOC_SECTION: - pe_dll_enable_reloc_section = 1; - break; -+ case OPTION_DISABLE_RELOC_SECTION: -+ pe_dll_enable_reloc_section = 0; -+ /* fall through */ -+ case OPTION_NO_DYNAMIC_BASE: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; -+ break; - case OPTION_FORCE_INTEGRITY: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; - break; - case OPTION_NX_COMPAT: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; - break; -+ case OPTION_NO_NX_COMPAT: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; -+ break; - case OPTION_NO_ISOLATION: - pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; - break; ---- a/ld/emultempl/pep.em -+++ b/ld/emultempl/pep.em -@@ -237,9 +240,12 @@ - OPTION_ENABLE_LONG_SECTION_NAMES, - OPTION_DISABLE_LONG_SECTION_NAMES, - OPTION_HIGH_ENTROPY_VA, -+ OPTION_NO_HIGH_ENTROPY_VA, - OPTION_DYNAMIC_BASE, -+ OPTION_NO_DYNAMIC_BASE, - OPTION_FORCE_INTEGRITY, - OPTION_NX_COMPAT, -+ OPTION_NO_NX_COMPAT, - OPTION_NO_ISOLATION, - OPTION_NO_SEH, - OPTION_NO_BIND, -@@ -248,7 +254,8 @@ - OPTION_NO_INSERT_TIMESTAMP, - OPTION_TERMINAL_SERVER_AWARE, - OPTION_BUILD_ID, -- OPTION_ENABLE_RELOC_SECTION -+ OPTION_ENABLE_RELOC_SECTION, -+ OPTION_DISABLE_RELOC_SECTION - }; - - static void -@@ -315,9 +322,12 @@ - {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, - {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, - {"high-entropy-va", no_argument, NULL, OPTION_HIGH_ENTROPY_VA}, -+ {"no-high-entropy-va", no_argument, NULL, OPTION_NO_HIGH_ENTROPY_VA}, - {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, -+ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, - {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, - {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, -+ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, - {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, - {"no-seh", no_argument, NULL, OPTION_NO_SEH}, - {"no-bind", no_argument, NULL, OPTION_NO_BIND}, -@@ -327,6 +337,7 @@ - {"no-insert-timestamp", no_argument, NULL, OPTION_NO_INSERT_TIMESTAMP}, - {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, - {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, -+ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, - {NULL, no_argument, NULL, 0} - }; - -@@ -448,11 +461,15 @@ - in object files\n")); - fprintf (file, _(" --high-entropy-va Image is compatible with 64-bit address space\n\ - layout randomization (ASLR)\n")); -+ fprintf (file, _(" --no-high-entropy-va Image is not compatible with 64-bit ASLR\n")); - fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ - address space layout randomization (ASLR)\n")); -+ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); - fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); -+ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); - fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); - fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); -+ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); - fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); - fprintf (file, _(" --no-seh Image does not use SEH; no SE handler may\n\ - be called in this image\n")); -@@ -809,12 +826,24 @@ - case OPTION_ENABLE_RELOC_SECTION: - pep_dll_enable_reloc_section = 1; - break; -+ case OPTION_DISABLE_RELOC_SECTION: -+ pep_dll_enable_reloc_section = 0; -+ /* fall through */ -+ case OPTION_NO_DYNAMIC_BASE: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; -+ /* fall through */ -+ case OPTION_NO_HIGH_ENTROPY_VA: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA; -+ break; - case OPTION_FORCE_INTEGRITY: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; - break; - case OPTION_NX_COMPAT: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; - break; -+ case OPTION_NO_NX_COMPAT: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; -+ break; - case OPTION_NO_ISOLATION: - pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; - break; diff --git a/contrib/guix/patches/gcc-10-remap-guix-store.patch b/contrib/guix/patches/gcc-10-remap-guix-store.patch new file mode 100644 index 000000000000..a47ef7a2df13 --- /dev/null +++ b/contrib/guix/patches/gcc-10-remap-guix-store.patch @@ -0,0 +1,25 @@ +From aad25427e74f387412e8bc9a9d7bbc6c496c792f Mon Sep 17 00:00:00 2001 +From: Andrew Chow +Date: Wed, 6 Jul 2022 16:49:41 -0400 +Subject: [PATCH] guix: remap guix store paths to /usr + +--- + libgcc/Makefile.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/libgcc/Makefile.in b/libgcc/Makefile.in +index 851e7657d07..476c2becd1c 100644 +--- a/libgcc/Makefile.in ++++ b/libgcc/Makefile.in +@@ -854,7 +854,7 @@ endif + # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and + # LIB2ADDEHSHARED matter. (Usually all three are identical.) + +-c_flags := -fexceptions ++c_flags := -fexceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + + ifeq ($(enable_shared),yes) + +-- +2.37.0 + diff --git a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch b/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch deleted file mode 100644 index f327c464f3ec..000000000000 --- a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch +++ /dev/null @@ -1,400 +0,0 @@ -guix: repro: Sort find output in libtool for gcc-8 - -Otherwise the resulting .a static libraries (e.g. libstdc++.a) will not -be reproducible and end up making the Bitcoin binaries non-reproducible -as well. - -See: https://reproducible-builds.org/docs/archives/#gnu-libtool - -diff --git a/gcc/configure b/gcc/configure -index 97ba7d7d69c..e37a96f0c0c 100755 ---- a/gcc/configure -+++ b/gcc/configure -@@ -19720,20 +19720,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libcc1/configure b/libcc1/configure -index f53a121611c..5740ca90cab 100755 ---- a/libcc1/configure -+++ b/libcc1/configure -@@ -12221,20 +12221,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libffi/configure b/libffi/configure -index 790a291011f..54b1ac18306 100755 ---- a/libffi/configure -+++ b/libffi/configure -@@ -12661,20 +12661,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libgo/config/libtool.m4 b/libgo/config/libtool.m4 -index f7005947454..8a84417b828 100644 ---- a/libgo/config/libtool.m4 -+++ b/libgo/config/libtool.m4 -@@ -6010,20 +6010,20 @@ if test "$_lt_caught_CXX_error" != yes; then - _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libgo/config/ltmain.sh b/libgo/config/ltmain.sh -index ce66b44906a..0f81c401407 100644 ---- a/libgo/config/ltmain.sh -+++ b/libgo/config/ltmain.sh -@@ -2917,7 +2917,7 @@ func_extract_archives () - darwin_file= - darwin_files= - for darwin_file in $darwin_filelist; do -- darwin_files=`find unfat-$$ -name $darwin_file -print | $NL2SP` -+ darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` - $LIPO -create -output "$darwin_file" $darwin_files - done # $darwin_filelist - $RM -rf unfat-$$ -@@ -2932,7 +2932,7 @@ func_extract_archives () - func_extract_an_archive "$my_xdir" "$my_xabs" - ;; - esac -- my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | $NL2SP` -+ my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` - done - - func_extract_archives_result="$my_oldobjs" -diff --git a/libhsail-rt/configure b/libhsail-rt/configure -index a4fcc10c1f9..8e671229fcd 100755 ---- a/libhsail-rt/configure -+++ b/libhsail-rt/configure -@@ -12244,20 +12244,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libitm/configure b/libitm/configure -index dbf386db434..29d4f10611f 100644 ---- a/libitm/configure -+++ b/libitm/configure -@@ -13067,20 +13067,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/liboffloadmic/configure b/liboffloadmic/configure -index f873716991b..7aa9186b10e 100644 ---- a/liboffloadmic/configure -+++ b/liboffloadmic/configure -@@ -12379,20 +12379,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/liboffloadmic/plugin/configure b/liboffloadmic/plugin/configure -index c031eb3e7fa..67fc7368f21 100644 ---- a/liboffloadmic/plugin/configure -+++ b/liboffloadmic/plugin/configure -@@ -12086,20 +12086,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libsanitizer/configure b/libsanitizer/configure -index 4695bc7d4f7..cb7d25c07e6 100755 ---- a/libsanitizer/configure -+++ b/libsanitizer/configure -@@ -13308,20 +13308,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libstdc++-v3/configure b/libstdc++-v3/configure -index 61457e940ec..21ef1f61e41 100755 ---- a/libstdc++-v3/configure -+++ b/libstdc++-v3/configure -@@ -13087,20 +13087,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libtool.m4 b/libtool.m4 -index 24d13f34409..940faaa161d 100644 ---- a/libtool.m4 -+++ b/libtool.m4 -@@ -6005,20 +6005,20 @@ if test "$_lt_caught_CXX_error" != yes; then - _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libvtv/configure b/libvtv/configure -index a197f750453..31ab3a0637b 100755 ---- a/libvtv/configure -+++ b/libvtv/configure -@@ -13339,20 +13339,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/ltmain.sh b/ltmain.sh -index 9503ec85d70..79f9ba89af5 100644 ---- a/ltmain.sh -+++ b/ltmain.sh -@@ -2917,7 +2917,7 @@ func_extract_archives () - darwin_file= - darwin_files= - for darwin_file in $darwin_filelist; do -- darwin_files=`find unfat-$$ -name $darwin_file -print | $NL2SP` -+ darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` - $LIPO -create -output "$darwin_file" $darwin_files - done # $darwin_filelist - $RM -rf unfat-$$ -@@ -2932,7 +2932,7 @@ func_extract_archives () - func_extract_an_archive "$my_xdir" "$my_xabs" - ;; - esac -- my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | $NL2SP` -+ my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` - done - - func_extract_archives_result="$my_oldobjs" diff --git a/contrib/guix/patches/gcc-broken-longjmp.patch b/contrib/guix/patches/gcc-broken-longjmp.patch new file mode 100644 index 000000000000..1cfc0918b090 --- /dev/null +++ b/contrib/guix/patches/gcc-broken-longjmp.patch @@ -0,0 +1,68 @@ +commit eb5698897c52702498938592d7f76e67d126451f +Author: Eric Botcazou +Date: Wed May 5 22:48:51 2021 +0200 + + Fix PR target/100402 + + This is a regression for 64-bit Windows present from mainline down to the 9 + branch and introduced by the fix for PR target/99234. Again SEH, but with + a twist related to the way MinGW implements setjmp/longjmp, which turns out + to be piggybacked on SEH with recent versions of MinGW, i.e. the longjmp + performs a bona-fide unwinding of the stack, because it calls RtlUnwindEx + with the second argument initially passed to setjmp, which is the result of + __builtin_frame_address (0) in the MinGW header file: + + define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) + + This means that we directly expose the frame pointer to the SEH machinery + here (unlike with regular exception handling where we use an intermediate + CFA) and thus that we cannot do whatever we want with it. The old code + would leave it unaligned, i.e. not multiple of 16, whereas the new code + aligns it, but this breaks for some reason; at least it appears that a + .seh_setframe directive with 0 as second argument always works, so the + fix aligns it this way. + + gcc/ + PR target/100402 + * config/i386/i386.c (ix86_compute_frame_layout): For a SEH target, + always return the establisher frame for __builtin_frame_address (0). + gcc/testsuite/ + * gcc.c-torture/execute/20210505-1.c: New test. + +diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c +index 2f838840e96..06ad1b2274e 100644 +--- a/gcc/config/i386/i386.c ++++ b/gcc/config/i386/i386.c +@@ -6356,12 +6356,29 @@ ix86_compute_frame_layout (void) + area, see the SEH code in config/i386/winnt.c for the rationale. */ + frame->hard_frame_pointer_offset = frame->sse_reg_save_offset; + +- /* If we can leave the frame pointer where it is, do so. Also, return ++ /* If we can leave the frame pointer where it is, do so; however return + the establisher frame for __builtin_frame_address (0) or else if the +- frame overflows the SEH maximum frame size. */ ++ frame overflows the SEH maximum frame size. ++ ++ Note that the value returned by __builtin_frame_address (0) is quite ++ constrained, because setjmp is piggybacked on the SEH machinery with ++ recent versions of MinGW: ++ ++ # elif defined(__SEH__) ++ # if defined(__aarch64__) || defined(_ARM64_) ++ # define setjmp(BUF) _setjmp((BUF), __builtin_sponentry()) ++ # elif (__MINGW_GCC_VERSION < 40702) ++ # define setjmp(BUF) _setjmp((BUF), mingw_getsp()) ++ # else ++ # define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) ++ # endif ++ ++ and the second argument passed to _setjmp, if not null, is forwarded ++ to the TargetFrame parameter of RtlUnwindEx by longjmp (after it has ++ built an ExceptionRecord on the fly describing the setjmp buffer). */ + const HOST_WIDE_INT diff + = frame->stack_pointer_offset - frame->hard_frame_pointer_offset; +- if (diff <= 255) ++ if (diff <= 255 && !crtl->accesses_prior_frames) + { + /* The resulting diff will be a multiple of 16 lower than 255, + i.e. at most 240 as required by the unwind data structure. */ diff --git a/contrib/guix/patches/glibc-2.24-fcommon.patch b/contrib/guix/patches/glibc-2.24-fcommon.patch new file mode 100644 index 000000000000..2bc32ede9056 --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-fcommon.patch @@ -0,0 +1,32 @@ +commit 264a4a0dbe1f4369db315080034b500bed66016c +Author: fanquake +Date: Fri May 6 11:03:04 2022 +0100 + + build: use -fcommon to retain legacy behaviour with GCC 10 + + GCC 10 started using -fno-common by default, which causes issues with + the powerpc builds using gibc 2.24. A patch was commited to glibc to fix + the issue, 18363b4f010da9ba459b13310b113ac0647c2fcc but is non-trvial + to backport, and was broken in at least one way, see the followup in + commit 7650321ce037302bfc2f026aa19e0213b8d02fe6. + + For now, retain the legacy GCC behaviour by passing -fcommon when + building glibc. + + https://gcc.gnu.org/onlinedocs/gcc/Code-Gen-Options.html. + https://sourceware.org/git/?p=glibc.git;a=commit;h=18363b4f010da9ba459b13310b113ac0647c2fcc + https://sourceware.org/git/?p=glibc.git;a=commit;h=7650321ce037302bfc2f026aa19e0213b8d02fe6 + +diff --git a/Makeconfig b/Makeconfig +index ee379f5852..63c4a2f234 100644 +--- a/Makeconfig ++++ b/Makeconfig +@@ -824,7 +824,7 @@ ifeq "$(strip $(+cflags))" "" + +cflags := $(default_cflags) + endif # $(+cflags) == "" + +-+cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags) +++cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags) -fcommon + +gcc-nowarn := -w + + # Don't duplicate options if we inherited variables from the parent. diff --git a/contrib/guix/patches/glibc-2.24-guix-prefix.patch b/contrib/guix/patches/glibc-2.24-guix-prefix.patch new file mode 100644 index 000000000000..875e8cd61191 --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-guix-prefix.patch @@ -0,0 +1,25 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +We might be able to drop this in favour of using --with-nonshared-cflags +when we being using newer versions of glibc. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -950,6 +950,10 @@ object-suffixes-for-libc += .oS + # shared objects. We don't want to use CFLAGS-os because users may, for + # example, make that processor-specific. + CFLAGS-.oS = $(CFLAGS-.o) $(PIC-ccflag) ++ ++# Map Guix store paths to /usr ++CFLAGS-.oS += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + CPPFLAGS-.oS = $(CPPFLAGS-.o) -DPIC -DLIBC_NONSHARED=1 + libtype.oS = lib%_nonshared.a + endif +-- +2.35.1 + diff --git a/contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch b/contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch new file mode 100644 index 000000000000..16a595d613c1 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch @@ -0,0 +1,87 @@ +commit 78a90c2f74a2012dd3eff302189e47ff6779a757 +Author: Andreas Schwab +Date: Fri Mar 2 23:07:14 2018 +0100 + + Fix multiple definitions of __nss_*_database (bug 22918) + + (cherry picked from commit eaf6753f8aac33a36deb98c1031d1bad7b593d2d) + +diff --git a/nscd/gai.c b/nscd/gai.c +index d081747797..576fd0045b 100644 +--- a/nscd/gai.c ++++ b/nscd/gai.c +@@ -45,3 +45,6 @@ + #ifdef HAVE_LIBIDN + # include + #endif ++ ++/* Some variables normally defined in libc. */ ++service_user *__nss_hosts_database attribute_hidden; +diff --git a/nss/nsswitch.c b/nss/nsswitch.c +index d5e655974f..b0f0c11a3e 100644 +--- a/nss/nsswitch.c ++++ b/nss/nsswitch.c +@@ -62,7 +62,7 @@ static service_library *nss_new_service (name_database *database, + + /* Declare external database variables. */ + #define DEFINE_DATABASE(name) \ +- extern service_user *__nss_##name##_database attribute_hidden; \ ++ service_user *__nss_##name##_database attribute_hidden; \ + weak_extern (__nss_##name##_database) + #include "databases.def" + #undef DEFINE_DATABASE +diff --git a/nss/nsswitch.h b/nss/nsswitch.h +index eccb535ef5..63573b9ebc 100644 +--- a/nss/nsswitch.h ++++ b/nss/nsswitch.h +@@ -226,10 +226,10 @@ libc_hidden_proto (__nss_hostname_digits_dots) + #define MAX_NR_ADDRS 48 + + /* Prototypes for __nss_*_lookup2 functions. */ +-#define DEFINE_DATABASE(arg) \ +- service_user *__nss_##arg##_database attribute_hidden; \ +- int __nss_##arg##_lookup2 (service_user **, const char *, \ +- const char *, void **); \ ++#define DEFINE_DATABASE(arg) \ ++ extern service_user *__nss_##arg##_database attribute_hidden; \ ++ int __nss_##arg##_lookup2 (service_user **, const char *, \ ++ const char *, void **); \ + libc_hidden_proto (__nss_##arg##_lookup2) + #include "databases.def" + #undef DEFINE_DATABASE +diff --git a/posix/tst-rfc3484-2.c b/posix/tst-rfc3484-2.c +index f509534ca9..8c64ac59ff 100644 +--- a/posix/tst-rfc3484-2.c ++++ b/posix/tst-rfc3484-2.c +@@ -58,6 +58,7 @@ _res_hconf_init (void) + #undef USE_NSCD + #include "../sysdeps/posix/getaddrinfo.c" + ++service_user *__nss_hosts_database attribute_hidden; + + /* This is the beginning of the real test code. The above defines + (among other things) the function rfc3484_sort. */ +diff --git a/posix/tst-rfc3484-3.c b/posix/tst-rfc3484-3.c +index ae44087a10..1c61aaf844 100644 +--- a/posix/tst-rfc3484-3.c ++++ b/posix/tst-rfc3484-3.c +@@ -58,6 +58,7 @@ _res_hconf_init (void) + #undef USE_NSCD + #include "../sysdeps/posix/getaddrinfo.c" + ++service_user *__nss_hosts_database attribute_hidden; + + /* This is the beginning of the real test code. The above defines + (among other things) the function rfc3484_sort. */ +diff --git a/posix/tst-rfc3484.c b/posix/tst-rfc3484.c +index 7f191abbbc..8f45848e44 100644 +--- a/posix/tst-rfc3484.c ++++ b/posix/tst-rfc3484.c +@@ -58,6 +58,7 @@ _res_hconf_init (void) + #undef USE_NSCD + #include "../sysdeps/posix/getaddrinfo.c" + ++service_user *__nss_hosts_database attribute_hidden; + + /* This is the beginning of the real test code. The above defines + (among other things) the function rfc3484_sort. */ diff --git a/contrib/guix/patches/glibc-2.27-guix-prefix.patch b/contrib/guix/patches/glibc-2.27-guix-prefix.patch new file mode 100644 index 000000000000..d777af74f018 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-guix-prefix.patch @@ -0,0 +1,25 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +We might be able to drop this in favour of using --with-nonshared-cflags +when we being using newer versions of glibc. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -992,6 +992,10 @@ object-suffixes := + CPPFLAGS-.o = $(pic-default) + # libc.a must be compiled with -fPIE/-fpie for static PIE. + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) ++ ++# Map Guix store paths to /usr ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) +-- +2.35.1 + diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch similarity index 90% rename from contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch rename to contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch index d6217157ee57..c0f8495c41de 100644 --- a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch @@ -1,3 +1,7 @@ +Note that this has been modified from the original commit, to use __has_include +instead of __has_include__, as the later was causing build failures with GCC 10. +See also: http://lists.busybox.net/pipermail/buildroot/2020-July/590376.html. + https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 @@ -59,7 +63,7 @@ index d612ef4c6c..0b2042620b 100644 #include #include -#include -+#if __has_include__ () ++#if __has_include () +# include +#else +# include diff --git a/contrib/guix/patches/lief-fix-ppc64-nx-default.patch b/contrib/guix/patches/lief-fix-ppc64-nx-default.patch new file mode 100644 index 000000000000..101bc1ddc0cf --- /dev/null +++ b/contrib/guix/patches/lief-fix-ppc64-nx-default.patch @@ -0,0 +1,29 @@ +Correct default for Binary::has_nx on ppc64 + +From the Linux kernel source: + + * This is the default if a program doesn't have a PT_GNU_STACK + * program header entry. The PPC64 ELF ABI has a non executable stack + * stack by default, so in the absence of a PT_GNU_STACK program header + * we turn execute permission off. + +This patch can be dropped the next time we update LIEF. + +diff --git a/src/ELF/Binary.cpp b/src/ELF/Binary.cpp +index a90be1ab..fd2d9764 100644 +--- a/src/ELF/Binary.cpp ++++ b/src/ELF/Binary.cpp +@@ -1084,7 +1084,12 @@ bool Binary::has_nx() const { + return segment->type() == SEGMENT_TYPES::PT_GNU_STACK; + }); + if (it_stack == std::end(segments_)) { +- return false; ++ if (header().machine_type() == ARCH::EM_PPC64) { ++ // The PPC64 ELF ABI has a non-executable stack by default. ++ return true; ++ } else { ++ return false; ++ } + } + + return !(*it_stack)->has(ELF_SEGMENT_FLAGS::PF_X); diff --git a/contrib/guix/patches/nsis-SConstruct-sde-support.patch b/contrib/guix/patches/nsis-SConstruct-sde-support.patch deleted file mode 100644 index f58406a7a089..000000000000 --- a/contrib/guix/patches/nsis-SConstruct-sde-support.patch +++ /dev/null @@ -1,18 +0,0 @@ -https://github.com/kichik/nsis/pull/13 -https://sourceforge.net/p/nsis/code/7248/ - -diff --git a/SConstruct b/SConstruct -index e8252c9..41786f2 100755 ---- a/SConstruct -+++ b/SConstruct -@@ -95,8 +95,8 @@ default_doctype = 'html' - if defenv.WhereIs('hhc', os.environ['PATH']): - default_doctype = 'chm' - --from time import strftime, gmtime --cvs_version = strftime('%d-%b-%Y.cvs', gmtime()) -+import time -+cvs_version = time.strftime('%d-%b-%Y.cvs', time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) - - opts = Variables() - diff --git a/contrib/guix/patches/nsis-disable-installer-reloc.patch b/contrib/guix/patches/nsis-disable-installer-reloc.patch new file mode 100644 index 000000000000..4914527e56be --- /dev/null +++ b/contrib/guix/patches/nsis-disable-installer-reloc.patch @@ -0,0 +1,30 @@ +Patch NSIS so that it's installer stubs, produced at NSIS build time, +do not contain .reloc sections, which will exist by default when using +binutils/ld 2.36+. + +This ultimately fixes an issue when running the installer with the +"Force randomization for images (Mandatory ASLR)" setting active. + +This patch has not yet been sent upstream, because it's not clear if this +is the best fix, for the underlying issue, which seems to be that makensis +doesn't account for .reloc sections when it builds installers. + +The existence of a reloc section shouldn't be a problem, and, if anything, +is actually a requirement for working ASLR. All other Windows binaries we +produce contain them, and function correctly when under the same +"Force randomization for images (Mandatory ASLR)" setting. + +See: +https://github.com/bitcoin/bitcoin/issues/25726 +https://sourceforge.net/p/nsis/bugs/1131/ + +--- a/SCons/Config/gnu ++++ b/SCons/Config/gnu +@@ -102,6 +102,7 @@ stub_env.Append(LINKFLAGS = ['-mwindows']) # build windows executables + stub_env.Append(LINKFLAGS = ['$NODEFLIBS_FLAG']) # no standard libraries + stub_env.Append(LINKFLAGS = ['$ALIGN_FLAG']) # 512 bytes align + stub_env.Append(LINKFLAGS = ['$MAP_FLAG']) # generate map file ++stub_env.Append(LINKFLAGS = ['-Wl,--disable-reloc-section']) + + conf = FlagsConfigure(stub_env) + conf.CheckCompileFlag('-fno-tree-loop-distribute-patterns') # GCC 10: Don't generate msvcrt!memmove calls (bug #1248) diff --git a/contrib/guix/patches/nsis-gcc-10-memmove.patch b/contrib/guix/patches/nsis-gcc-10-memmove.patch new file mode 100644 index 000000000000..a1aadfd4f361 --- /dev/null +++ b/contrib/guix/patches/nsis-gcc-10-memmove.patch @@ -0,0 +1,23 @@ +commit f6df41524e703dc471e283e566a48e05a735b7f2 +Author: Anders +Date: Sat Jun 27 23:18:45 2020 +0000 + + Don't let GCC 10 generate memmove calls (bug #1248) + + git-svn-id: https://svn.code.sf.net/p/nsis/code/NSIS/trunk@7189 212acab6-be3b-0410-9dea-997c60f758d6 + +diff --git a/SCons/Config/gnu b/SCons/Config/gnu +index bfcb362d..21fa446b 100644 +--- a/SCons/Config/gnu ++++ b/SCons/Config/gnu +@@ -103,6 +103,10 @@ stub_env.Append(LINKFLAGS = ['$NODEFLIBS_FLAG']) # no standard libraries + stub_env.Append(LINKFLAGS = ['$ALIGN_FLAG']) # 512 bytes align + stub_env.Append(LINKFLAGS = ['$MAP_FLAG']) # generate map file + ++conf = FlagsConfigure(stub_env) ++conf.CheckCompileFlag('-fno-tree-loop-distribute-patterns') # GCC 10: Don't generate msvcrt!memmove calls (bug #1248) ++conf.Finish() ++ + stub_uenv = stub_env.Clone() + stub_uenv.Append(CPPDEFINES = ['_UNICODE', 'UNICODE']) + diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch new file mode 100644 index 000000000000..072f76eafd38 --- /dev/null +++ b/contrib/guix/patches/vmov-alignment.patch @@ -0,0 +1,267 @@ +Description: Use unaligned VMOV instructions +Author: Stephen Kitt +Bug-Debian: https://bugs.debian.org/939559 + +Based on a patch originally by Claude Heiland-Allen + +--- a/gcc/config/i386/sse.md ++++ b/gcc/config/i386/sse.md +@@ -1058,17 +1058,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (mode))) + { +- if (misaligned_operand (operands[1], mode)) +- return "vmovu\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmova\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovu\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + else + { +- if (misaligned_operand (operands[1], mode)) +- return "vmovdqu\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmovdqa\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovdqu\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -1184,17 +1178,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (mode))) + { +- if (misaligned_operand (operands[0], mode)) +- return "vmovu\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmova\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovu\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + else + { +- if (misaligned_operand (operands[0], mode)) +- return "vmovdqu\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmovdqa\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovdqu\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -7806,7 +7794,7 @@ + "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))" + "@ + %vmovlps\t{%1, %0|%q0, %1} +- %vmovaps\t{%1, %0|%0, %1} ++ %vmovups\t{%1, %0|%0, %1} + %vmovlps\t{%1, %d0|%d0, %q1}" + [(set_attr "type" "ssemov") + (set_attr "prefix" "maybe_vex") +@@ -13997,29 +13985,15 @@ + switch (mode) + { + case E_V8DFmode: +- if (misaligned_operand (operands[2], mode)) +- return "vmovupd\t{%2, %x0|%x0, %2}"; +- else +- return "vmovapd\t{%2, %x0|%x0, %2}"; ++ return "vmovupd\t{%2, %x0|%x0, %2}"; + case E_V16SFmode: +- if (misaligned_operand (operands[2], mode)) +- return "vmovups\t{%2, %x0|%x0, %2}"; +- else +- return "vmovaps\t{%2, %x0|%x0, %2}"; ++ return "vmovups\t{%2, %x0|%x0, %2}"; + case E_V8DImode: +- if (misaligned_operand (operands[2], mode)) +- return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa64\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + case E_V16SImode: +- if (misaligned_operand (operands[2], mode)) +- return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa32\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + default: + gcc_unreachable (); + } +@@ -21225,63 +21199,27 @@ + switch (get_attr_mode (insn)) + { + case MODE_V16SF: +- if (misaligned_operand (operands[1], mode)) +- return "vmovups\t{%1, %t0|%t0, %1}"; +- else +- return "vmovaps\t{%1, %t0|%t0, %1}"; ++ return "vmovups\t{%1, %t0|%t0, %1}"; + case MODE_V8DF: +- if (misaligned_operand (operands[1], mode)) +- return "vmovupd\t{%1, %t0|%t0, %1}"; +- else +- return "vmovapd\t{%1, %t0|%t0, %1}"; ++ return "vmovupd\t{%1, %t0|%t0, %1}"; + case MODE_V8SF: +- if (misaligned_operand (operands[1], mode)) +- return "vmovups\t{%1, %x0|%x0, %1}"; +- else +- return "vmovaps\t{%1, %x0|%x0, %1}"; ++ return "vmovups\t{%1, %x0|%x0, %1}"; + case MODE_V4DF: +- if (misaligned_operand (operands[1], mode)) +- return "vmovupd\t{%1, %x0|%x0, %1}"; +- else +- return "vmovapd\t{%1, %x0|%x0, %1}"; ++ return "vmovupd\t{%1, %x0|%x0, %1}"; + case MODE_XI: +- if (misaligned_operand (operands[1], mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (mode) == 8) +- return "vmovdqu64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqu32\t{%1, %t0|%t0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %t0|%t0, %1}"; ++ else if (GET_MODE_SIZE (mode) == 8) ++ return "vmovdqu64\t{%1, %t0|%t0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (mode) == 8) +- return "vmovdqa64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqa32\t{%1, %t0|%t0, %1}"; +- } ++ return "vmovdqu32\t{%1, %t0|%t0, %1}"; + case MODE_OI: +- if (misaligned_operand (operands[1], mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (mode) == 8) +- return "vmovdqu64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqu32\t{%1, %x0|%x0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %x0|%x0, %1}"; ++ else if (GET_MODE_SIZE (mode) == 8) ++ return "vmovdqu64\t{%1, %x0|%x0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (mode) == 8) +- return "vmovdqa64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqa32\t{%1, %x0|%x0, %1}"; +- } ++ return "vmovdqu32\t{%1, %x0|%x0, %1}"; + default: + gcc_unreachable (); + } +--- a/gcc/config/i386/i386.c ++++ b/gcc/config/i386/i386.c +@@ -4981,13 +4981,13 @@ + switch (type) + { + case opcode_int: +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + break; + case opcode_float: +- opcode = misaligned_p ? "vmovups" : "vmovaps"; ++ opcode = "vmovups"; + break; + case opcode_double: +- opcode = misaligned_p ? "vmovupd" : "vmovapd"; ++ opcode = "vmovupd"; + break; + } + } +@@ -4996,16 +4996,16 @@ + switch (scalar_mode) + { + case E_SFmode: +- opcode = misaligned_p ? "%vmovups" : "%vmovaps"; ++ opcode = "%vmovups"; + break; + case E_DFmode: +- opcode = misaligned_p ? "%vmovupd" : "%vmovapd"; ++ opcode = "%vmovupd"; + break; + case E_TFmode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + default: + gcc_unreachable (); +@@ -5017,48 +5017,32 @@ + { + case E_QImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "%vmovdqu"; + break; + case E_HImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "%vmovdqu"; + break; + case E_SImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_DImode: + case E_TImode: + case E_OImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_XImode: +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + break; + default: + gcc_unreachable (); diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index 4037936404ba..2850c4b993ef 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,7 +20,7 @@ expand_path() { cd "${1}" && pwd -P } -BDB_PREFIX="$(expand_path ${1})/db4"; shift; +BDB_PREFIX="$(expand_path "${1}")/db4"; shift; BDB_VERSION='db-4.8.30.NC' BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" @@ -55,13 +55,22 @@ http_get() { echo "File ${2} already exists; not downloading again" elif check_exists curl; then curl --insecure --retry 5 "${1}" -o "${2}" - else + elif check_exists wget; then wget --no-check-certificate "${1}" -O "${2}" + else + echo "Simple transfer utilities 'curl' and 'wget' not found. Please install one of them and try again." + exit 1 fi sha256_check "${3}" "${2}" } +# Ensure the commands we use exist on the system +if ! check_exists patch; then + echo "Command-line tool 'patch' not found. Install patch and try again." + exit 1 +fi + mkdir -p "${BDB_PREFIX}" http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" @@ -221,10 +230,10 @@ EOF # The packaged config.guess and config.sub are ancient (2009) and can cause build issues. # Replace them with modern versions. # See https://github.com/bitcoin/bitcoin/issues/16064 -CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=55eaf3e779455c4e5cc9f82efb5278be8f8f900b' -CONFIG_GUESS_HASH='2d1ff7bca773d2ec3c6217118129220fa72d8adda67c7d2bf79994b3129232c1' -CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=55eaf3e779455c4e5cc9f82efb5278be8f8f900b' -CONFIG_SUB_HASH='3a4befde9bcdf0fdb2763fc1bfa74e8696df94e1ad7aac8042d133c8ff1d2e32' +CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' +CONFIG_GUESS_HASH='c8f530e01840719871748a8071113435bdfdf75b74c57e78e47898edea8754ae' +CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' +CONFIG_SUB_HASH='3969f7d5f6967ccc6f792401b8ef3916a1d1b1d0f0de5a4e354c95addb8b800e' rm -f "dist/config.guess" rm -f "dist/config.sub" diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 73f54cd4885e..b72c7b0d0885 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -2,7 +2,7 @@ # # linearize-data.py: Construct a linear, no-fork version of the chain. # -# Copyright (c) 2013-2020 The Bitcoin Core developers +# Copyright (c) 2013-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -17,53 +17,12 @@ import time import glob from collections import namedtuple -from binascii import unhexlify settings = {} -def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() - -def uint32(x): - return x & 0xffffffff - -def bytereverse(x): - return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | - (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) - -def bufreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - word = struct.unpack('@I', in_buf[i:i+4])[0] - out_words.append(struct.pack('@I', bytereverse(word))) - return b''.join(out_words) - -def wordreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - out_words.append(in_buf[i:i+4]) - out_words.reverse() - return b''.join(out_words) - -def calc_hdr_hash(blk_hdr): - hash1 = hashlib.sha256() - hash1.update(blk_hdr) - hash1_o = hash1.digest() - - hash2 = hashlib.sha256() - hash2.update(hash1_o) - hash2_o = hash2.digest() - - return hash2_o - def calc_hash_str(blk_hdr): - hash = calc_hdr_hash(blk_hdr) - hash = bufreverse(hash) - hash = wordreverse(hash) - hash_str = hash.hex() - return hash_str + blk_hdr_hash = hashlib.sha256(hashlib.sha256(blk_hdr).digest()).digest() + return blk_hdr_hash[::-1].hex() def get_blk_dt(blk_hdr): members = struct.unpack(" - - - - - - - - - - - - - - - PACKAGE_NAME - - - - - diff --git a/contrib/macdeploy/background.tiff b/contrib/macdeploy/background.tiff new file mode 100644 index 000000000000..1fb088c8374a Binary files /dev/null and b/contrib/macdeploy/background.tiff differ diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh deleted file mode 100755 index d481413cc382..000000000000 --- a/contrib/macdeploy/detached-sig-apply.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# Copyright (c) 2014-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -set -e - -UNSIGNED="$1" -SIGNATURE="$2" -ROOTDIR=dist -OUTDIR=signed-app -SIGNAPPLE=signapple - -if [ -z "$UNSIGNED" ]; then - echo "usage: $0 " - exit 1 -fi - -if [ -z "$SIGNATURE" ]; then - echo "usage: $0 " - exit 1 -fi - -${SIGNAPPLE} apply ${UNSIGNED} ${SIGNATURE} -mv ${ROOTDIR} ${OUTDIR} -echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 4f246cbb3fe4..f393331084e3 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,9 +8,11 @@ set -e ROOTDIR=dist BUNDLE="${ROOTDIR}/Bitcoin-Qt.app" +BINARY="${BUNDLE}/Contents/MacOS/Bitcoin-Qt" SIGNAPPLE=signapple TEMPDIR=sign.temp -OUT=signature-osx.tar.gz +ARCH=$(${SIGNAPPLE} info ${BINARY} | head -n 1 | cut -d " " -f 1) +OUT="signature-osx-${ARCH}.tar.gz" OUTROOT=osx/dist if [ -z "$1" ]; then diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index 457d8f5e645c..6efaaccb8e16 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -8,6 +8,21 @@ import gzip import os import contextlib +# monkey-patch Python 3.8 and older to fix wrong TAR header handling +# see https://github.com/bitcoin/bitcoin/pull/24534 +# and https://github.com/python/cpython/pull/18080 for more info +if sys.version_info < (3, 9): + _old_create_header = tarfile.TarInfo._create_header + def _create_header(info, format, encoding, errors): + buf = _old_create_header(info, format, encoding, errors) + # replace devmajor/devminor with binary zeroes + buf = buf[:329] + bytes(16) + buf[345:] + # recompute checksum + chksum = tarfile.calc_chksums(buf)[0] + buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] + return buf + tarfile.TarInfo._create_header = staticmethod(_create_header) + @contextlib.contextmanager def cd(path): """Context manager that restores PWD even if an exception was raised.""" @@ -75,14 +90,21 @@ def run(): tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) if tarinfo.linkname and tarinfo.linkname.startswith("./"): tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname)) + # make metadata deterministic + tarinfo.mtime = 0 + tarinfo.uid, tarinfo.uname = 0, '' + tarinfo.gid, tarinfo.gname = 0, '' + # don't use isdir() as there are also executable files present + tarinfo.mode = 0o0755 if tarinfo.mode & 0o0100 else 0o0644 return tarinfo with cd(dir_to_add): + # recursion already adds entries in sorted order tarfp.add(".", recursive=True, filter=change_tarinfo_base) print("Creating output .tar.gz file...") with out_sdktgz_path.open("wb") as fp: - with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf: - with tarfile.open(mode="w", fileobj=gzf) as tarfp: + with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf: + with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp: print("Adding MacOSX SDK {} files...".format(sdk_version)) tarfp_add_with_base_change(tarfp, sdk_dir, out_name) print("Adding libc++ headers...") diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 055a932eee49..2420539b7cad 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -16,7 +16,7 @@ # along with this program. If not, see . # -import sys, re, os, shutil, stat, os.path +import sys, re, os, platform, shutil, stat, subprocess, os.path from argparse import ArgumentParser from ds_store import DSStore from mac_alias import Alias @@ -211,7 +211,7 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: return libraries def runInstallNameTool(action: str, *args): - installnametoolbin=os.getenv("INSTALLNAMETOOL", "install_name_tool") + installnametoolbin=os.getenv("INSTALL_NAME_TOOL", "install_name_tool") run([installnametoolbin, "-"+action] + list(args), check=True) def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int): @@ -244,49 +244,47 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional fromPath = framework.sourceFilePath toDir = os.path.join(path, framework.destinationDirectory) toPath = os.path.join(toDir, framework.binaryName) - - if not os.path.exists(fromPath): - raise RuntimeError(f"No file at {fromPath}") - - if os.path.exists(toPath): - return None # Already there - - if not os.path.exists(toDir): - os.makedirs(toDir) - - shutil.copy2(fromPath, toPath) - if verbose: - print("Copied:", fromPath) - print(" to:", toPath) + + if framework.isDylib(): + if not os.path.exists(fromPath): + raise RuntimeError(f"No file at {fromPath}") + + if os.path.exists(toPath): + return None # Already there + + if not os.path.exists(toDir): + os.makedirs(toDir) + + shutil.copy2(fromPath, toPath) + if verbose: + print("Copied:", fromPath) + print(" to:", toPath) + else: + to_dir = os.path.join(path, "Contents", "Frameworks", framework.frameworkName) + if os.path.exists(to_dir): + return None # Already there + + from_dir = framework.frameworkPath + if not os.path.exists(from_dir): + raise RuntimeError(f"No directory at {from_dir}") + + shutil.copytree(from_dir, to_dir, symlinks=True) + if verbose: + print("Copied:", from_dir) + print(" to:", to_dir) + + headers_link = os.path.join(to_dir, "Headers") + if os.path.exists(headers_link): + os.unlink(headers_link) + + headers_dir = os.path.join(to_dir, framework.binaryDirectory, "Headers") + if os.path.exists(headers_dir): + shutil.rmtree(headers_dir) permissions = os.stat(toPath) if not permissions.st_mode & stat.S_IWRITE: os.chmod(toPath, permissions.st_mode | stat.S_IWRITE) - if not framework.isDylib(): # Copy resources for real frameworks - - linkfrom = os.path.join(path, "Contents","Frameworks", framework.frameworkName, "Versions", "Current") - linkto = framework.version - if not os.path.exists(linkfrom): - os.symlink(linkto, linkfrom) - print("Linked:", linkfrom, "->", linkto) - fromResourcesDir = framework.sourceResourcesDirectory - if os.path.exists(fromResourcesDir): - toResourcesDir = os.path.join(path, framework.destinationResourcesDirectory) - shutil.copytree(fromResourcesDir, toResourcesDir, symlinks=True) - if verbose: - print("Copied resources:", fromResourcesDir) - print(" to:", toResourcesDir) - fromContentsDir = framework.sourceVersionContentsDirectory - if not os.path.exists(fromContentsDir): - fromContentsDir = framework.sourceContentsDirectory - if os.path.exists(fromContentsDir): - toContentsDir = os.path.join(path, framework.destinationVersionContentsDirectory) - shutil.copytree(fromContentsDir, toContentsDir, symlinks=True) - if verbose: - print("Copied Contents:", fromContentsDir) - print(" to:", toContentsDir) - return toPath def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPath: str, strip: bool, verbose: int, deploymentInfo: Optional[DeploymentInfo] = None) -> DeploymentInfo: @@ -543,6 +541,25 @@ ds.close() # ------------------------------------------------ +if platform.system() == "Darwin": + subprocess.check_call(f"codesign --deep --force --sign - {target}", shell=True) + +print("+ Installing background.tiff +") + +bg_path = os.path.join('dist', '.background', 'background.tiff') +os.mkdir(os.path.dirname(bg_path)) + +tiff_path = os.path.join('contrib', 'macdeploy', 'background.tiff') +shutil.copy2(tiff_path, bg_path) + +# ------------------------------------------------ + +print("+ Generating symlink for /Applications +") + +os.symlink("/Applications", os.path.join('dist', "Applications")) + +# ------------------------------------------------ + if config.dmg is not None: print("+ Preparing .dmg disk image +") @@ -566,19 +583,6 @@ if config.dmg is not None: print("Attaching temp image...") output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout - m = re.search(r"/Volumes/(.+$)", output) - disk_root = m.group(0) - - print("+ Applying fancy settings +") - - bg_path = os.path.join(disk_root, ".background", os.path.basename('background.tiff')) - os.mkdir(os.path.dirname(bg_path)) - if verbose: - print('background.tiff', "->", bg_path) - shutil.copy2('background.tiff', bg_path) - - os.symlink("/Applications", os.path.join(disk_root, "Applications")) - print("+ Finalizing .dmg disk image +") run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) diff --git a/contrib/message-capture/message-capture-parser.py b/contrib/message-capture/message-capture-parser.py index 9988478f1b90..33759ee71377 100755 --- a/contrib/message-capture/message-capture-parser.py +++ b/contrib/message-capture/message-capture-parser.py @@ -79,7 +79,7 @@ def to_jsonable(obj: Any) -> Any: val = getattr(obj, slot, None) if slot in HASH_INTS and isinstance(val, int): ret[slot] = ser_uint256(val).hex() - elif slot in HASH_INT_VECTORS and isinstance(val[0], int): + elif slot in HASH_INT_VECTORS and all(isinstance(a, int) for a in val): ret[slot] = [ser_uint256(a).hex() for a in val] else: ret[slot] = to_jsonable(val) diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh index 1cde19efd1a4..7ebcbf225175 100755 --- a/contrib/qos/tc.sh +++ b/contrib/qos/tc.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/seeds/.gitignore b/contrib/seeds/.gitignore index e4a39d609343..d9a2451f70fa 100644 --- a/contrib/seeds/.gitignore +++ b/contrib/seeds/.gitignore @@ -1 +1,2 @@ seeds_main.txt +asmap-filled.dat diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index 3bca094d3b9d..b2ea7522ac9f 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -8,20 +8,11 @@ and remove old versions as necessary (at a minimum when GetDesirableServiceFlags changes its default return value, as those are the services which seeds are added to addrman with). -The seeds compiled into the release are created from sipa's DNS seed data, like this: +The seeds compiled into the release are created from sipa's DNS seed and AS map +data. Run the following commands from the `/contrib/seeds` directory: - curl -s http://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt - python3 makeseeds.py < seeds_main.txt > nodes_main.txt + curl https://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt + curl https://bitcoin.sipa.be/asmap-filled.dat > asmap-filled.dat + python3 makeseeds.py -a asmap-filled.dat < seeds_main.txt > nodes_main.txt + cat nodes_main_manual.txt >> nodes_main.txt python3 generate-seeds.py . > ../../src/chainparamsseeds.h - -## Dependencies - -Ubuntu, Debian: - - sudo apt-get install python3-dnspython - -and/or for other operating systems: - - pip install dnspython - -See https://dnspython.readthedocs.io/en/latest/installation.html for more information. diff --git a/contrib/seeds/asmap.py b/contrib/seeds/asmap.py new file mode 100644 index 000000000000..e28e5cf532e9 --- /dev/null +++ b/contrib/seeds/asmap.py @@ -0,0 +1,815 @@ +# Copyright (c) 2022 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +""" +This module provides the ASNEntry and ASMap classes. +""" + +import copy +import ipaddress +import random +import unittest +from enum import Enum +from functools import total_ordering +from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union, overload + +def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> List[bool]: + """ + Convert an IPv4 or IPv6 network to a prefix represented as a list of bits. + + IPv4 ranges are remapped to their IPv4-mapped IPv6 range (::ffff:0:0/96). + """ + num_bits = net.prefixlen + netrange = int.from_bytes(net.network_address.packed, 'big') + + # Map an IPv4 prefix into IPv6 space. + if isinstance(net, ipaddress.IPv4Network): + num_bits += 96 + netrange += 0xffff00000000 + + # Strip unused bottom bits. + assert (netrange & ((1 << (128 - num_bits)) - 1)) == 0 + return [((netrange >> (127 - i)) & 1) != 0 for i in range(num_bits)] + +def prefix_to_net(prefix: List[bool]) -> Union[ipaddress.IPv4Network,ipaddress.IPv6Network]: + """The reverse operation of net_to_prefix.""" + # Convert to number + netrange = sum(b << (127 - i) for i, b in enumerate(prefix)) + num_bits = len(prefix) + assert num_bits <= 128 + + # Return IPv4 range if in ::ffff:0:0/96 + if num_bits >= 96 and (netrange >> 32) == 0xffff: + return ipaddress.IPv4Network((netrange & 0xffffffff, num_bits - 96), True) + + # Return IPv6 range otherwise. + return ipaddress.IPv6Network((netrange, num_bits), True) + +# Shortcut for (prefix, ASN) entries. +ASNEntry = Tuple[List[bool], int] + +# Shortcut for (prefix, old ASN, new ASN) entries. +ASNDiff = Tuple[List[bool], int, int] + +class _VarLenCoder: + """ + A class representing a custom variable-length binary encoder/decoder for + integers. Each object represents a different coder, with different parameters + minval and clsbits. + + The encoding is easiest to describe using an example. Let's say minval=100 and + clsbits=[4,2,2,3]. In that case: + - x in [100..115]: encoded as [0] + [4-bit BE encoding of (x-100)]. + - x in [116..119]: encoded as [1,0] + [2-bit BE encoding of (x-116)]. + - x in [120..123]: encoded as [1,1,0] + [2-bit BE encoding of (x-120)]. + - x in [124..131]: encoded as [1,1,1] + [3-bit BE encoding of (x-124)]. + + In general, every number is encoded as: + - First, k "1"-bits, where k is the class the number falls in (there is one class + per element of clsbits). + - Then, a "0"-bit, unless k is the highest class, in which case there is nothing. + - Lastly, clsbits[k] bits encoding in big endian the position in its class that + number falls into. + - Every class k consists of 2^clsbits[k] consecutive integers. k=0 starts at minval, + other classes start one past the last element of the class before it. + """ + + def __init__(self, minval: int, clsbits: List[int]): + """Construct a new _VarLenCoder.""" + self._minval = minval + self._clsbits = clsbits + self._maxval = minval + sum(1 << b for b in clsbits) - 1 + + def can_encode(self, val: int) -> bool: + """Check whether value val is in the range this coder supports.""" + return self._minval <= val <= self._maxval + + def encode(self, val: int, ret: List[int]) -> None: + """Append encoding of val onto integer list ret.""" + + assert self._minval <= val <= self._maxval + val -= self._minval + bits = 0 + for k, bits in enumerate(self._clsbits): + if val >> bits: + # If the value will not fit in class k, subtract its range from v, + # emit a "1" bit and continue with the next class. + val -= 1 << bits + ret.append(1) + else: + if k + 1 < len(self._clsbits): + # Unless we're in the last class, emit a "0" bit. + ret.append(0) + break + # And then encode v (now the position within the class) in big endian. + ret.extend((val >> (bits - 1 - b)) & 1 for b in range(bits)) + + def encode_size(self, val: int) -> int: + """Compute how many bits are needed to encode val.""" + assert self._minval <= val <= self._maxval + val -= self._minval + ret = 0 + bits = 0 + for k, bits in enumerate(self._clsbits): + if val >> bits: + val -= 1 << bits + ret += 1 + else: + ret += k + 1 < len(self._clsbits) + break + return ret + bits + + def decode(self, stream, bitpos) -> Tuple[int,int]: + """Decode a number starting at bitpos in stream, returning value and new bitpos.""" + val = self._minval + bits = 0 + for k, bits in enumerate(self._clsbits): + bit = 0 + if k + 1 < len(self._clsbits): + bit = stream[bitpos] + bitpos += 1 + if not bit: + break + val += 1 << bits + for i in range(bits): + bit = stream[bitpos] + bitpos += 1 + val += bit << (bits - 1 - i) + return val, bitpos + +# Variable-length encoders used in the binary asmap format. +_CODER_INS = _VarLenCoder(0, [0, 0, 1]) +_CODER_ASN = _VarLenCoder(1, list(range(15, 25))) +_CODER_MATCH = _VarLenCoder(2, list(range(1, 9))) +_CODER_JUMP = _VarLenCoder(17, list(range(5, 31))) + +class _Instruction(Enum): + """One instruction in the binary asmap format.""" + # A return instruction, encoded as [0], returns a constant ASN. It is followed by + # an integer using the ASN encoding. + RETURN = 0 + # A jump instruction, encoded as [1,0] inspects the next unused bit in the input + # and either continues execution (if 0), or skips a specified number of bits (if 1). + # It is followed by an integer, and then two subprograms. The integer uses jump encoding + # and corresponds to the length of the first subprogram (so it can be skipped). + JUMP = 1 + # A match instruction, encoded as [1,1,0] inspects 1 or more of the next unused bits + # in the input with its argument. If they all match, execution continues. If they do + # not, failure is returned. If a default instruction has been executed before, instead + # of failure the default instruction's argument is returned. It is followed by an + # integer in match encoding, and a subprogram. That value is at least 2 bits and at + # most 9 bits. An n-bit value signifies matching (n-1) bits in the input with the lower + # (n-1) bits in the match value. + MATCH = 2 + # A default instruction, encoded as [1,1,1] sets the default variable to its argument, + # and continues execution. It is followed by an integer in ASN encoding, and a subprogram. + DEFAULT = 3 + # Not an actual instruction, but a way to encode the empty program that fails. In the + # encoder, it is used more generally to represent the failure case inside MATCH instructions, + # which may (if used inside the context of a DEFAULT instruction) actually correspond to + # a successful return. In this usage, they're always converted to an actual MATCH or RETURN + # before the top level is reached (see make_default below). + END = 4 + +class _BinNode: + """A class representing a (node of) the parsed binary asmap format.""" + + @overload + def __init__(self, ins: _Instruction): ... + @overload + def __init__(self, ins: _Instruction, arg1: int): ... + @overload + def __init__(self, ins: _Instruction, arg1: "_BinNode", arg2: "_BinNode"): ... + @overload + def __init__(self, ins: _Instruction, arg1: int, arg2: "_BinNode"): ... + + def __init__(self, ins: _Instruction, arg1=None, arg2=None): + """ + Construct a new asmap node. Possibilities are: + - _BinNode(_Instruction.RETURN, asn) + - _BinNode(_Instruction.JUMP, node_0, node_1) + - _BinNode(_Instruction.MATCH, val, node) + - _BinNode(_Instruction.DEFAULT, asn, node) + - _BinNode(_Instruction.END) + """ + self.ins = ins + self.arg1 = arg1 + self.arg2 = arg2 + if ins == _Instruction.RETURN: + assert isinstance(arg1, int) + assert arg2 is None + self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1) + elif ins == _Instruction.JUMP: + assert isinstance(arg1, _BinNode) + assert isinstance(arg2, _BinNode) + self.size = (_CODER_INS.encode_size(ins.value) + _CODER_JUMP.encode_size(arg1.size) + + arg1.size + arg2.size) + elif ins == _Instruction.DEFAULT: + assert isinstance(arg1, int) + assert isinstance(arg2, _BinNode) + self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1) + arg2.size + elif ins == _Instruction.MATCH: + assert isinstance(arg1, int) + assert isinstance(arg2, _BinNode) + self.size = (_CODER_INS.encode_size(ins.value) + _CODER_MATCH.encode_size(arg1) + + arg2.size) + elif ins == _Instruction.END: + assert arg1 is None + assert arg2 is None + self.size = 0 + else: + assert False + + @staticmethod + def make_end() -> "_BinNode": + """Constructor for a _BinNode with just an END instruction.""" + return _BinNode(_Instruction.END) + + @staticmethod + def make_leaf(val: int) -> "_BinNode": + """Constructor for a _BinNode of just a RETURN instruction.""" + assert val is not None and val > 0 + return _BinNode(_Instruction.RETURN, val) + + @staticmethod + def make_branch(node0: "_BinNode", node1: "_BinNode") -> "_BinNode": + """ + Construct a _BinNode corresponding to running either the node0 or node1 subprogram, + based on the next input bit. It exploits shortcuts that are possible in the encoding, + and uses either a JUMP, MATCH, or END instruction. + """ + if node0.ins == _Instruction.END and node1.ins == _Instruction.END: + return node0 + if node0.ins == _Instruction.END: + if node1.ins == _Instruction.MATCH and node1.arg1 <= 0xFF: + return _BinNode(node1.ins, node1.arg1 + (1 << node1.arg1.bit_length()), node1.arg2) + return _BinNode(_Instruction.MATCH, 3, node1) + if node1.ins == _Instruction.END: + if node0.ins == _Instruction.MATCH and node0.arg1 <= 0xFF: + return _BinNode(node0.ins, node0.arg1 + (1 << (node0.arg1.bit_length() - 1)), + node0.arg2) + return _BinNode(_Instruction.MATCH, 2, node0) + return _BinNode(_Instruction.JUMP, node0, node1) + + @staticmethod + def make_default(val: int, sub: "_BinNode") -> "_BinNode": + """ + Construct a _BinNode that corresponds to the specified subprogram, with the specified + default value. It exploits shortcuts that are possible in the encoding, and will use + either a DEFAULT or a RETURN instruction.""" + assert val is not None and val > 0 + if sub.ins == _Instruction.END: + return _BinNode(_Instruction.RETURN, val) + if sub.ins in (_Instruction.RETURN, _Instruction.DEFAULT): + return sub + return _BinNode(_Instruction.DEFAULT, val, sub) + +@total_ordering +class ASMap: + """ + A class whose objects represent a mapping from subnets to ASNs. + + Internally the mapping is stored as a binary trie, but can be converted + from/to a list of ASNEntry objects, and from/to the binary asmap file format. + + In the trie representation, nodes are represented as bare lists for efficiency + and ease of manipulation: + - [0] means an unassigned subnet (no ASN mapping for it is present) + - [int] means a subnet mapped entirely to the specified ASN. + - [node,node] means a subnet whose lower half and upper half have different + - mappings, represented by new trie nodes. + """ + + def update(self, prefix: List[bool], asn: int) -> None: + """Update this ASMap object to map prefix to the specified asn.""" + assert asn == 0 or _CODER_ASN.can_encode(asn) + + def recurse(node: List, offset: int) -> None: + if offset == len(prefix): + # Reached the end of prefix; overwrite this node. + node.clear() + node.append(asn) + return + if len(node) == 1: + # Need to descend into a leaf node; split it up. + oldasn = node[0] + node.clear() + node.append([oldasn]) + node.append([oldasn]) + # Descend into the node. + recurse(node[prefix[offset]], offset + 1) + # If the result is two identical leaf children, merge them. + if len(node[0]) == 1 and len(node[1]) == 1 and node[0] == node[1]: + oldasn = node[0][0] + node.clear() + node.append(oldasn) + recurse(self._trie, 0) + + def update_multi(self, entries: List[Tuple[List[bool], int]]) -> None: + """Apply multiple update operations, where longer prefixes take precedence.""" + entries.sort(key=lambda entry: len(entry[0])) + for prefix, asn in entries: + self.update(prefix, asn) + + def _set_trie(self, trie) -> None: + """Set trie directly. Internal use only.""" + def recurse(node: List) -> None: + if len(node) < 2: + return + recurse(node[0]) + recurse(node[1]) + if len(node[0]) == 2: + return + if node[0] == node[1]: + if len(node[0]) == 0: + node.clear() + else: + asn = node[0][0] + node.clear() + node.append(asn) + recurse(trie) + self._trie = trie + + def __init__(self, entries: Optional[Iterable[ASNEntry]] = None) -> None: + """Construct an ASMap object from an optional list of entries.""" + self._trie = [0] + if entries is not None: + def entry_key(entry): + """Sort function that places shorter prefixes first.""" + prefix, asn = entry + return len(prefix), prefix, asn + for prefix, asn in sorted(entries, key=entry_key): + self.update(prefix, asn) + + def lookup(self, prefix: List[bool]) -> Optional[int]: + """Look up a prefix. Returns ASN, or 0 if unassigned, or None if indeterminate.""" + node = self._trie + for bit in prefix: + if len(node) == 1: + break + node = node[bit] + if len(node) == 1: + return node[0] + return None + + def _to_entries_flat(self, fill: bool = False) -> List[ASNEntry]: + """Convert an ASMap object to a list of non-overlapping (prefix, asn) objects.""" + prefix : List[bool] = [] + + def recurse(node: List) -> List[ASNEntry]: + ret = [] + if len(node) == 1: + if node[0] > 0: + ret = [(list(prefix), node[0])] + elif len(node) == 2: + prefix.append(False) + ret = recurse(node[0]) + prefix[-1] = True + ret += recurse(node[1]) + prefix.pop() + if fill and len(ret) > 1: + asns = set(x[1] for x in ret) + if len(asns) == 1: + ret = [(list(prefix), list(asns)[0])] + return ret + return recurse(self._trie) + + def _to_entries_minimal(self, fill: bool = False) -> List[ASNEntry]: + """Convert a trie to a minimal list of ASNEntry objects, exploiting overlap.""" + prefix : List[bool] = [] + + def recurse(node: List) -> (Tuple[Dict[Optional[int], List[ASNEntry]], bool]): + if len(node) == 1 and node[0] == 0: + return {None if fill else 0: []}, True + if len(node) == 1: + return {node[0]: [], None: [(list(prefix), node[0])]}, False + ret: Dict[Optional[int], List[ASNEntry]] = {} + prefix.append(False) + left, lhole = recurse(node[0]) + prefix[-1] = True + right, rhole = recurse(node[1]) + prefix.pop() + hole = not fill and (lhole or rhole) + def candidate(ctx: Optional[int], res0: Optional[List[ASNEntry]], + res1: Optional[List[ASNEntry]]): + if res0 is not None and res1 is not None: + if ctx not in ret or len(res0) + len(res1) < len(ret[ctx]): + ret[ctx] = res0 + res1 + for ctx in set(left) | set(right): + candidate(ctx, left.get(ctx), right.get(ctx)) + candidate(ctx, left.get(None), right.get(ctx)) + candidate(ctx, left.get(ctx), right.get(None)) + if not hole: + for ctx in list(ret): + if ctx is not None: + candidate(None, [(list(prefix), ctx)], ret[ctx]) + if None in ret: + ret = {ctx:entries for ctx, entries in ret.items() + if ctx is None or len(entries) < len(ret[None])} + if hole: + ret = {ctx:entries for ctx, entries in ret.items() if ctx is None or ctx == 0} + return ret, hole + res, _ = recurse(self._trie) + return res[0] if 0 in res else res[None] + + def __str__(self) -> str: + """Convert this ASMap object to a string containing Python code constructing it.""" + return f"ASMap({self._trie})" + + def to_entries(self, overlapping: bool = True, fill: bool = False) -> List[ASNEntry]: + """ + Convert the mappings in this ASMap object to a list of ASNEntry objects. + + Arguments: + overlapping: Permit the subnets in the resulting ASNEntry to overlap. + Setting this can result in a shorter list. + fill: Permit the resulting ASNEntry objects to cover subnets that + are unassigned in this ASMap object. Setting this can + result in a shorter list. + """ + if overlapping: + return self._to_entries_minimal(fill) + return self._to_entries_flat(fill) + + @staticmethod + def from_random(num_leaves: int = 10, max_asn: int = 6, + unassigned_prob: float = 0.5) -> "ASMap": + """ + Construct a random ASMap object, with specified: + - Number of leaves in its trie (at least 1) + - Maximum ASN value (at least 1) + - Probability for leaf nodes to be unassigned + + The number of leaves in the resulting object may be less than what is + requested. This method is mostly intended for testing. + """ + assert num_leaves >= 1 + assert max_asn >= 1 or unassigned_prob == 1 + assert _CODER_ASN.can_encode(max_asn) + assert 0.0 <= unassigned_prob <= 1.0 + trie: List = [] + leaves = [trie] + ret = ASMap() + for i in range(1, num_leaves): + idx = random.randrange(i) + leaf = leaves[idx] + lastleaf = leaves.pop() + if idx + 1 < i: + leaves[idx] = lastleaf + leaf.append([]) + leaf.append([]) + leaves.append(leaf[0]) + leaves.append(leaf[1]) + for leaf in leaves: + if random.random() >= unassigned_prob: + leaf.append(random.randrange(1, max_asn + 1)) + else: + leaf.append(0) + #pylint: disable=protected-access + ret._set_trie(trie) + return ret + + def _to_binnode(self, fill: bool = False) -> _BinNode: + """Convert a trie to a _BinNode object.""" + def recurse(node: List) -> Tuple[Dict[Optional[int], _BinNode], bool]: + if len(node) == 1 and node[0] == 0: + return {(None if fill else 0): _BinNode.make_end()}, True + if len(node) == 1: + return {None: _BinNode.make_leaf(node[0]), node[0]: _BinNode.make_end()}, False + ret: Dict[Optional[int], _BinNode] = {} + left, lhole = recurse(node[0]) + right, rhole = recurse(node[1]) + hole = (lhole or rhole) and not fill + + def candidate(ctx: Optional[int], arg1, arg2, func: Callable): + if arg1 is not None and arg2 is not None: + cand = func(arg1, arg2) + if ctx not in ret or cand.size < ret[ctx].size: + ret[ctx] = cand + + for ctx in set(left) | set(right): + candidate(ctx, left.get(ctx), right.get(ctx), _BinNode.make_branch) + candidate(ctx, left.get(None), right.get(ctx), _BinNode.make_branch) + candidate(ctx, left.get(ctx), right.get(None), _BinNode.make_branch) + if not hole: + for ctx in set(ret) - set([None]): + candidate(None, ctx, ret[ctx], _BinNode.make_default) + if None in ret: + ret = {ctx:enc for ctx, enc in ret.items() + if ctx is None or enc.size < ret[None].size} + if hole: + ret = {ctx:enc for ctx, enc in ret.items() if ctx is None or ctx == 0} + return ret, hole + res, _ = recurse(self._trie) + return res[0] if 0 in res else res[None] + + @staticmethod + def _from_binnode(binnode: _BinNode) -> "ASMap": + """Construct an ASMap object from a _BinNode. Internal use only.""" + def recurse(node: _BinNode, default: int) -> List: + if node.ins == _Instruction.RETURN: + return [node.arg1] + if node.ins == _Instruction.JUMP: + return [recurse(node.arg1, default), recurse(node.arg2, default)] + if node.ins == _Instruction.MATCH: + val = node.arg1 + sub = recurse(node.arg2, default) + while val >= 2: + bit = val & 1 + val >>= 1 + if bit: + sub = [[default], sub] + else: + sub = [sub, [default]] + return sub + assert node.ins == _Instruction.DEFAULT + return recurse(node.arg2, node.arg1) + ret = ASMap() + if binnode.ins != _Instruction.END: + #pylint: disable=protected-access + ret._set_trie(recurse(binnode, 0)) + return ret + + def to_binary(self, fill: bool = False) -> bytes: + """ + Convert this ASMap object to binary. + + Argument: + fill: permit the resulting binary encoder to contain mappers for + unassigned subnets in this ASMap object. Doing so may + reduce the size of the encoding. + Returns: + A bytes object with the encoding of this ASMap object. + """ + bits: List[int] = [] + + def recurse(node: _BinNode) -> None: + _CODER_INS.encode(node.ins.value, bits) + if node.ins == _Instruction.RETURN: + _CODER_ASN.encode(node.arg1, bits) + elif node.ins == _Instruction.JUMP: + _CODER_JUMP.encode(node.arg1.size, bits) + recurse(node.arg1) + recurse(node.arg2) + elif node.ins == _Instruction.DEFAULT: + _CODER_ASN.encode(node.arg1, bits) + recurse(node.arg2) + else: + assert node.ins == _Instruction.MATCH + _CODER_MATCH.encode(node.arg1, bits) + recurse(node.arg2) + + binnode = self._to_binnode(fill) + if binnode.ins != _Instruction.END: + recurse(binnode) + + val = 0 + nbits = 0 + ret = [] + for bit in bits: + val += (bit << nbits) + nbits += 1 + if nbits == 8: + ret.append(val) + val = 0 + nbits = 0 + if nbits: + ret.append(val) + return bytes(ret) + + @staticmethod + def from_binary(bindata: bytes) -> Optional["ASMap"]: + """Decode an ASMap object from the provided binary encoding.""" + + bits: List[int] = [] + for byte in bindata: + bits.extend((byte >> i) & 1 for i in range(8)) + + def recurse(bitpos: int) -> Tuple[_BinNode, int]: + insval, bitpos = _CODER_INS.decode(bits, bitpos) + ins = _Instruction(insval) + if ins == _Instruction.RETURN: + asn, bitpos = _CODER_ASN.decode(bits, bitpos) + return _BinNode(ins, asn), bitpos + if ins == _Instruction.JUMP: + jump, bitpos = _CODER_JUMP.decode(bits, bitpos) + left, bitpos1 = recurse(bitpos) + if bitpos1 != bitpos + jump: + raise ValueError("Inconsistent jump") + right, bitpos = recurse(bitpos1) + return _BinNode(ins, left, right), bitpos + if ins == _Instruction.MATCH: + match, bitpos = _CODER_MATCH.decode(bits, bitpos) + sub, bitpos = recurse(bitpos) + return _BinNode(ins, match, sub), bitpos + assert ins == _Instruction.DEFAULT + asn, bitpos = _CODER_ASN.decode(bits, bitpos) + sub, bitpos = recurse(bitpos) + return _BinNode(ins, asn, sub), bitpos + + if len(bits) == 0: + binnode = _BinNode(_Instruction.END) + else: + try: + binnode, bitpos = recurse(0) + except (ValueError, IndexError): + return None + if bitpos < len(bits) - 7: + return None + if not all(bit == 0 for bit in bits[bitpos:]): + return None + + return ASMap._from_binnode(binnode) + + def __lt__(self, other: "ASMap") -> bool: + return self._trie < other._trie + + def __eq__(self, other: object) -> bool: + if isinstance(other, ASMap): + return self._trie == other._trie + return False + + def extends(self, req: "ASMap") -> bool: + """Determine whether this matches req for all subranges where req is assigned.""" + def recurse(actual: List, require: List) -> bool: + if len(require) == 1 and require[0] == 0: + return True + if len(require) == 1: + if len(actual) == 1: + return bool(require[0] == actual[0]) + return recurse(actual[0], require) and recurse(actual[1], require) + if len(actual) == 2: + return recurse(actual[0], require[0]) and recurse(actual[1], require[1]) + return recurse(actual, require[0]) and recurse(actual, require[1]) + assert isinstance(req, ASMap) + #pylint: disable=protected-access + return recurse(self._trie, req._trie) + + def diff(self, other: "ASMap") -> List[ASNDiff]: + """Compute the diff from self to other.""" + prefix: List[bool] = [] + ret: List[ASNDiff] = [] + + def recurse(old_node: List, new_node: List): + if len(old_node) == 1 and len(new_node) == 1: + if old_node[0] != new_node[0]: + ret.append((list(prefix), old_node[0], new_node[0])) + else: + old_left: List = old_node if len(old_node) == 1 else old_node[0] + old_right: List = old_node if len(old_node) == 1 else old_node[1] + new_left: List = new_node if len(new_node) == 1 else new_node[0] + new_right: List = new_node if len(new_node) == 1 else new_node[1] + prefix.append(False) + recurse(old_left, new_left) + prefix[-1] = True + recurse(old_right, new_right) + prefix.pop() + assert isinstance(other, ASMap) + #pylint: disable=protected-access + recurse(self._trie, other._trie) + return ret + + def __copy__(self) -> "ASMap": + """Construct a copy of this ASMap object. Its state will not be shared.""" + ret = ASMap() + #pylint: disable=protected-access + ret._set_trie(copy.deepcopy(self._trie)) + return ret + + def __deepcopy__(self, _) -> "ASMap": + # ASMap objects do not allow sharing of the _trie member, so we don't need the memoization. + return self.__copy__() + + +class TestASMap(unittest.TestCase): + """Unit tests for this module.""" + + def test_ipv6_prefix_roundtrips(self) -> None: + """Test that random IPv6 network ranges roundtrip through prefix encoding.""" + for _ in range(20): + net_bits = random.getrandbits(128) + for prefix_len in range(0, 129): + masked_bits = (net_bits >> (128 - prefix_len)) << (128 - prefix_len) + net = ipaddress.IPv6Network((masked_bits.to_bytes(16, 'big'), prefix_len)) + prefix = net_to_prefix(net) + self.assertTrue(len(prefix) <= 128) + net2 = prefix_to_net(prefix) + self.assertEqual(net, net2) + + def test_ipv4_prefix_roundtrips(self) -> None: + """Test that random IPv4 network ranges roundtrip through prefix encoding.""" + for _ in range(100): + net_bits = random.getrandbits(32) + for prefix_len in range(0, 33): + masked_bits = (net_bits >> (32 - prefix_len)) << (32 - prefix_len) + net = ipaddress.IPv4Network((masked_bits.to_bytes(4, 'big'), prefix_len)) + prefix = net_to_prefix(net) + self.assertTrue(32 <= len(prefix) <= 128) + net2 = prefix_to_net(prefix) + self.assertEqual(net, net2) + + def test_asmap_roundtrips(self) -> None: + """Test case that verifies random ASMap objects roundtrip to/from entries/binary.""" + # Iterate over the number of leaves the random test ASMap objects have. + for leaves in range(1, 20): + # Iterate over the number of bits in the AS numbers used. + for asnbits in range(0, 24): + # Iterate over the probability that leaves are unassigned. + for pct in range(101): + # Construct a random ASMap object according to the above parameters. + asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits), + unassigned_prob=0.01 * pct) + # Run tests for to_entries and construction from those entries, both + # for overlapping and non-overlapping ones. + for overlapping in [False, True]: + entries = asmap.to_entries(overlapping=overlapping, fill=False) + random.shuffle(entries) + asmap2 = ASMap(entries) + assert asmap2 is not None + self.assertEqual(asmap2, asmap) + entries = asmap.to_entries(overlapping=overlapping, fill=True) + random.shuffle(entries) + asmap2 = ASMap(entries) + assert asmap2 is not None + self.assertTrue(asmap2.extends(asmap)) + + # Run tests for to_binary and construction from binary. + enc = asmap.to_binary(fill=False) + asmap3 = ASMap.from_binary(enc) + assert asmap3 is not None + self.assertEqual(asmap3, asmap) + enc = asmap.to_binary(fill=True) + asmap3 = ASMap.from_binary(enc) + assert asmap3 is not None + self.assertTrue(asmap3.extends(asmap)) + + def test_patching(self) -> None: + """Test behavior of update, lookup, extends, and diff.""" + #pylint: disable=too-many-locals,too-many-nested-blocks + # Iterate over the number of leaves the random test ASMap objects have. + for leaves in range(1, 20): + # Iterate over the number of bits in the AS numbers used. + for asnbits in range(0, 10): + # Iterate over the probability that leaves are unassigned. + for pct in range(0, 101): + # Construct a random ASMap object according to the above parameters. + asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits), + unassigned_prob=0.01 * pct) + # Make a copy of that asmap object to which patches will be applied. + # It starts off being equal to asmap. + patched = copy.copy(asmap) + # Keep a list of patches performed. + patches: List[ASNEntry] = [] + # Initially there cannot be any difference. + self.assertEqual(asmap.diff(patched), []) + # Make 5 patches, each building on top of the previous ones. + for _ in range(0, 5): + # Construct a random path and new ASN to assign it to, apply it to patched, + # and remember it in patches. + pathlen = random.randrange(5) + path = [random.getrandbits(1) != 0 for _ in range(pathlen)] + newasn = random.randrange(1 + (1 << asnbits)) + patched.update(path, newasn) + patches = [(path, newasn)] + patches + + # Compute the diff, and whether asmap extends patched, and the other way + # around. + diff = asmap.diff(patched) + self.assertEqual(asmap == patched, len(diff) == 0) + extends = asmap.extends(patched) + back_extends = patched.extends(asmap) + # Determine whether those extends results are consistent with the diff + # result. + self.assertEqual(extends, all(d[2] == 0 for d in diff)) + self.assertEqual(back_extends, all(d[1] == 0 for d in diff)) + # For every diff found: + for path, old_asn, new_asn in diff: + # Verify asmap and patched actually differ there. + self.assertTrue(old_asn != new_asn) + self.assertEqual(asmap.lookup(path), old_asn) + self.assertEqual(patched.lookup(path), new_asn) + for _ in range(2): + # Extend the path far enough that it's smaller than any mapped + # range, and check the lookup holds there too. + spec_path = list(path) + while len(spec_path) < 32: + spec_path.append(random.getrandbits(1) != 0) + self.assertEqual(asmap.lookup(spec_path), old_asn) + self.assertEqual(patched.lookup(spec_path), new_asn) + # Search through the list of performed patches to find the last one + # applying to the extended path (note that patches is in reverse + # order, so the first match should work). + found = False + for patch_path, patch_asn in patches: + if spec_path[:len(patch_path)] == patch_path: + # When found, it must match whatever the result was patched + # to. + self.assertEqual(new_asn, patch_asn) + found = True + break + # And such a patch must exist. + self.assertTrue(found) + +if __name__ == '__main__': + unittest.main() diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index dbecba7d1db7..44345e39874c 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -61,7 +61,7 @@ def name_to_bip155(addr): raise ValueError(f'Invalid I2P {vchAddr}') elif '.' in addr: # IPv4 return (BIP155Network.IPV4, bytes((int(x) for x in addr.split('.')))) - elif ':' in addr: # IPv6 + elif ':' in addr: # IPv6 or CJDNS sub = [[], []] # prefix, suffix x = 0 addr = addr.split(':') @@ -77,7 +77,14 @@ def name_to_bip155(addr): sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) - return (BIP155Network.IPV6, bytes(sub[0] + ([0] * nullbytes) + sub[1])) + addr_bytes = bytes(sub[0] + ([0] * nullbytes) + sub[1]) + if addr_bytes[0] == 0xfc: + # Assume that seeds with fc00::/8 addresses belong to CJDNS, + # not to the publicly unroutable "Unique Local Unicast" network, see + # RFC4193: https://datatracker.ietf.org/doc/html/rfc4193#section-8 + return (BIP155Network.CJDNS, addr_bytes) + else: + return (BIP155Network.IPV6, addr_bytes) else: raise ValueError('Could not parse address %s' % addr) diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index 9be6a690a63d..eda58c370f29 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -1,31 +1,32 @@ #!/usr/bin/env python3 -# Copyright (c) 2013-2020 The Bitcoin Core developers +# Copyright (c) 2013-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # # Generate seeds.txt from Pieter's DNS seeder # +import argparse +import collections +import ipaddress import re import sys -import dns.resolver -import collections - -NSEEDS=512 +from typing import List, Dict, Union -MAX_SEEDS_PER_ASN=2 +from asmap import ASMap, net_to_prefix -MIN_BLOCKS = 337600 +NSEEDS=512 -# These are hosts that have been observed to be behaving strangely (e.g. -# aggressively connecting to every node). -with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f: - SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()} +MAX_SEEDS_PER_ASN = { + 'ipv4': 2, + 'ipv6': 10, +} +MIN_BLOCKS = 730000 PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") -PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$") +PATTERN_ONION = re.compile(r"^([a-z2-7]{56}\.onion):(\d+)$") PATTERN_AGENT = re.compile( r"^/Satoshi:(" r"0.14.(0|1|2|3|99)|" @@ -33,15 +34,22 @@ r"0.16.(0|1|2|3|99)|" r"0.17.(0|0.1|1|2|99)|" r"0.18.(0|1|99)|" - r"0.19.(0|1|99)|" - r"0.20.(0|1|99)|" - r"0.21.99" + r"0.19.(0|1|2|99)|" + r"0.20.(0|1|2|99)|" + r"0.21.(0|1|2|99)|" + r"22.(0|99)|" + r"23.(0|99)|" + r"24.99" r")") -def parseline(line): +def parseline(line: str) -> Union[dict, None]: + """ Parses a line from `seeds_main.txt` into a dictionary of details for that line. + or `None`, if the line could not be parsed. + """ sline = line.split() if len(sline) < 11: - return None + # line too short to be valid, skip it. + return None m = PATTERN_IPV4.match(sline[0]) sortkey = None ip = None @@ -105,98 +113,95 @@ def parseline(line): 'sortkey': sortkey, } -def dedup(ips): - '''deduplicate by address,port''' +def dedup(ips: List[Dict]) -> List[Dict]: + """ Remove duplicates from `ips` where multiple ips share address and port. """ d = {} for ip in ips: d[ip['ip'],ip['port']] = ip return list(d.values()) -def filtermultiport(ips): - '''Filter out hosts with more nodes per IP''' +def filtermultiport(ips: List[Dict]) -> List[Dict]: + """ Filter out hosts with more nodes per IP""" hist = collections.defaultdict(list) for ip in ips: hist[ip['sortkey']].append(ip) return [value[0] for (key,value) in list(hist.items()) if len(value)==1] -def lookup_asn(net, ip): - ''' - Look up the asn for an IP (4 or 6) address by querying cymru.com, or None - if it could not be found. - ''' - try: - if net == 'ipv4': - ipaddr = ip - prefix = '.origin' - else: # http://www.team-cymru.com/IP-ASN-mapping.html - res = str() # 2001:4860:b002:23::68 - for nb in ip.split(':')[:4]: # pick the first 4 nibbles - for c in nb.zfill(4): # right padded with '0' - res += c + '.' # 2001 4860 b002 0023 - ipaddr = res.rstrip('.') # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3 - prefix = '.origin6' - - asn = int([x.to_text() for x in dns.resolver.resolve('.'.join( - reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com', - 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) - return asn - except Exception: - sys.stderr.write('ERR: Could not resolve ASN for "' + ip + '"\n') - return None - # Based on Greg Maxwell's seed_filter.py -def filterbyasn(ips, max_per_asn, max_per_net): +def filterbyasn(asmap: ASMap, ips: List[Dict], max_per_asn: Dict, max_per_net: int) -> List[Dict]: + """ Prunes `ips` by + (a) trimming ips to have at most `max_per_net` ips from each net (e.g. ipv4, ipv6); and + (b) trimming ips to have at most `max_per_asn` ips from each asn in each net. + """ # Sift out ips by type ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']] ips_onion = [ip for ip in ips if ip['net'] == 'onion'] # Filter IPv46 by ASN, and limit to max_per_net per network result = [] - net_count = collections.defaultdict(int) - asn_count = collections.defaultdict(int) - for ip in ips_ipv46: + net_count: Dict[str, int] = collections.defaultdict(int) + asn_count: Dict[int, int] = collections.defaultdict(int) + + for i, ip in enumerate(ips_ipv46): if net_count[ip['net']] == max_per_net: + # do not add this ip as we already too many + # ips from this network continue - asn = lookup_asn(ip['net'], ip['ip']) - if asn is None or asn_count[asn] == max_per_asn: + asn = asmap.lookup(net_to_prefix(ipaddress.ip_network(ip['ip']))) + if not asn or asn_count[ip['net'], asn] == max_per_asn[ip['net']]: + # do not add this ip as we already have too many + # ips from this ASN on this network continue - asn_count[asn] += 1 + asn_count[ip['net'], asn] += 1 net_count[ip['net']] += 1 + ip['asn'] = asn result.append(ip) # Add back Onions (up to max_per_net) result.extend(ips_onion[0:max_per_net]) return result -def ip_stats(ips): - hist = collections.defaultdict(int) +def ip_stats(ips: List[Dict]) -> str: + """ Format and return pretty string from `ips`. """ + hist: Dict[str, int] = collections.defaultdict(int) for ip in ips: if ip is not None: hist[ip['net']] += 1 - return '%6d %6d %6d' % (hist['ipv4'], hist['ipv6'], hist['onion']) + return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d}" + +def parse_args(): + argparser = argparse.ArgumentParser(description='Generate a list of bitcoin node seed ip addresses.') + argparser.add_argument("-a","--asmap", help='the location of the asmap asn database file (required)', required=True) + return argparser.parse_args() def main(): + args = parse_args() + + print(f'Loading asmap database "{args.asmap}"…', end='', file=sys.stderr, flush=True) + with open(args.asmap, 'rb') as f: + asmap = ASMap.from_binary(f.read()) + print('Done.', file=sys.stderr) + + print('Loading and parsing DNS seeds…', end='', file=sys.stderr, flush=True) lines = sys.stdin.readlines() ips = [parseline(line) for line in lines] + print('Done.', file=sys.stderr) print('\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m', file=sys.stderr) - print('%s Initial' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Initial', file=sys.stderr) # Skip entries with invalid address. ips = [ip for ip in ips if ip is not None] - print('%s Skip entries with invalid address' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Skip entries with invalid address', file=sys.stderr) # Skip duplicates (in case multiple seeds files were concatenated) ips = dedup(ips) - print('%s After removing duplicates' % (ip_stats(ips)), file=sys.stderr) - # Skip entries from suspicious hosts. - ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS] - print('%s Skip entries from suspicious hosts' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} After removing duplicates', file=sys.stderr) # Enforce minimal number of blocks. ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS] - print('%s Enforce minimal number of blocks' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Enforce minimal number of blocks', file=sys.stderr) # Require service bit 1. ips = [ip for ip in ips if (ip['service'] & 1) == 1] - print('%s Require service bit 1' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require service bit 1', file=sys.stderr) # Require at least 50% 30-day uptime for clearnet, 10% for onion. req_uptime = { 'ipv4': 50, @@ -204,25 +209,28 @@ def main(): 'onion': 10, } ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]] - print('%s Require minimum uptime' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require minimum uptime', file=sys.stderr) # Require a known and recent user agent. ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])] - print('%s Require a known and recent user agent' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require a known and recent user agent', file=sys.stderr) # Sort by availability (and use last success as tie breaker) ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True) # Filter out hosts with multiple bitcoin ports, these are likely abusive ips = filtermultiport(ips) - print('%s Filter out hosts with multiple bitcoin ports' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Filter out hosts with multiple bitcoin ports', file=sys.stderr) # Look up ASNs and limit results, both per ASN and globally. - ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS) - print('%s Look up ASNs and limit results per ASN and per net' % (ip_stats(ips)), file=sys.stderr) + ips = filterbyasn(asmap, ips, MAX_SEEDS_PER_ASN, NSEEDS) + print(f'{ip_stats(ips):s} Look up ASNs and limit results per ASN and per net', file=sys.stderr) # Sort the results by IP address (for deterministic output). ips.sort(key=lambda x: (x['net'], x['sortkey'])) for ip in ips: if ip['net'] == 'ipv6': - print('[%s]:%i' % (ip['ip'], ip['port'])) + print(f"[{ip['ip']}]:{ip['port']}", end="") else: - print('%s:%i' % (ip['ip'], ip['port'])) + print(f"{ip['ip']}:{ip['port']}", end="") + if 'asn' in ip: + print(f" # AS{ip['asn']}", end="") + print() if __name__ == '__main__': main() diff --git a/contrib/seeds/nodes_main.txt b/contrib/seeds/nodes_main.txt index f7bfb6eb0a23..f8572b26c7f2 100644 --- a/contrib/seeds/nodes_main.txt +++ b/contrib/seeds/nodes_main.txt @@ -1,688 +1,857 @@ -2.39.173.126:8333 -3.14.168.201:48333 -4.36.112.44:8333 -5.8.18.31:8333 -5.14.200.167:8333 -5.56.20.2:8333 -5.102.146.99:8333 -5.103.137.146:9333 -5.128.87.126:8333 -5.133.65.82:8333 -5.187.55.242:8333 -5.188.62.24:8333 -5.188.62.33:8333 -5.199.133.193:8333 -8.38.89.152:8333 -13.231.20.249:8333 -18.27.79.17:8333 -20.184.15.116:8433 -23.28.205.97:8333 -23.106.252.230:8333 -23.175.0.202:8333 -23.175.0.212:8333 -23.241.250.252:8333 -23.245.24.154:8333 -24.86.184.66:8333 -24.116.246.9:8333 -24.141.34.166:8333 -24.155.196.246:8333 -24.157.130.222:8333 -24.188.176.255:8333 -24.237.70.53:8333 -27.124.4.67:8333 -31.17.70.80:8333 -31.21.8.32:8333 -31.45.118.10:8333 -31.132.17.56:8333 -31.134.121.223:8333 -32.214.183.114:8333 -35.137.236.32:8333 -35.185.145.105:8333 -35.209.51.212:8333 -35.245.175.76:8333 -37.116.95.41:8333 -37.143.9.107:8333 -37.143.116.43:8333 -37.191.244.149:8333 -37.211.78.253:8333 -37.221.209.222:24333 -37.228.92.110:8333 -43.225.62.107:8333 -43.225.157.152:8333 -45.36.184.6:8333 -45.48.168.16:8333 -45.85.85.8:8333 -45.85.85.9:8333 -45.129.180.214:8333 -45.149.78.128:8333 -45.151.125.218:8333 -45.154.255.46:8333 -45.155.157.239:8333 -46.28.132.34:8333 -46.28.204.21:8333 -46.32.50.98:8333 -46.59.13.35:8333 -46.128.40.173:8333 -46.128.140.193:8333 -46.146.248.89:8333 -46.166.162.45:20001 -46.188.15.6:8333 -46.229.165.142:8333 -46.229.238.187:8333 -46.249.83.82:8333 -46.254.217.169:8333 -47.74.191.34:8333 -47.115.53.163:8333 -47.187.26.135:8333 -47.222.103.234:8333 -47.253.5.99:8333 -49.232.82.76:8333 -49.247.215.43:8333 -50.2.13.166:8333 -50.34.39.72:8333 -50.45.232.189:8333 -50.68.104.92:8333 -51.68.36.57:8333 -51.154.60.34:8333 -52.169.238.66:8333 -54.197.30.223:8333 -54.227.66.57:8333 -58.158.0.86:8333 -58.171.135.242:8333 -58.229.208.158:8333 -60.244.109.19:8333 -62.38.75.208:8333 -62.74.143.11:8333 -62.80.227.49:8333 -62.152.58.16:9421 -62.210.167.199:8333 -62.234.188.160:8333 -62.251.54.163:8333 -63.227.116.162:8333 -65.19.155.82:8333 -65.95.49.102:8333 -66.18.172.21:8333 -66.240.237.155:8333 -67.210.228.203:8333 -69.30.215.42:8333 -69.59.18.206:8333 -69.64.33.71:8333 -69.119.193.9:8333 -69.209.23.72:8333 -70.123.125.237:8333 -70.185.56.136:8333 -71.38.90.235:8333 -72.12.73.70:8333 -72.53.134.182:8333 -72.225.7.80:8333 -72.234.182.39:8333 -72.250.184.57:8333 -73.83.103.79:8333 -74.118.137.119:8333 -74.133.100.74:8333 -74.215.219.214:8333 -74.220.255.190:8333 -75.158.39.231:8333 -77.53.53.196:8333 -77.70.16.245:8333 -77.105.87.97:8333 -77.120.113.69:8433 -77.120.122.22:8433 -77.166.83.167:8333 -77.247.178.130:8333 -78.27.139.13:8333 -78.63.28.146:8333 -78.83.103.4:8333 -78.141.123.99:8333 -79.77.33.131:8333 -79.77.133.30:8333 -79.101.1.25:8333 -79.117.192.229:8333 -79.133.228.55:8333 -79.146.21.163:8333 -80.89.203.172:8001 -80.93.213.246:8333 -80.192.98.110:8334 -80.229.28.60:8333 -80.232.247.210:8333 -80.242.39.76:8333 -80.253.94.252:8333 -81.0.198.25:8333 -81.7.13.84:8333 -81.117.225.245:8333 -81.135.137.225:8333 -81.171.22.143:8333 -81.191.233.134:8333 -81.232.78.75:8333 -81.242.91.23:8333 -82.29.58.109:8333 -82.136.99.22:8333 -82.149.97.25:17567 -82.165.19.48:8333 -82.194.153.233:8333 -82.197.215.125:8333 -82.199.102.10:8333 -82.200.205.30:8333 -82.202.68.231:8333 -82.221.128.31:8333 -82.228.6.131:8333 -83.85.139.94:8333 -83.99.245.20:8333 -83.137.41.10:8333 -83.174.209.87:8333 -83.217.8.31:44420 -84.38.3.249:8333 -84.38.185.122:8333 -84.92.92.247:8333 -84.192.16.234:8333 -84.194.158.124:8333 -84.212.145.24:8333 -84.212.244.95:8333 -84.216.51.36:8333 -84.255.249.163:8333 -85.25.255.147:8333 -85.70.156.209:8333 -85.145.142.46:8333 -85.170.233.95:8333 -85.184.138.108:8333 -85.190.0.5:8333 -85.191.200.51:8333 -85.192.191.6:18500 -85.194.238.131:8333 -85.195.54.110:8333 -85.214.161.252:8333 -85.214.185.51:8333 -85.241.106.203:8333 -85.246.168.252:8333 -86.56.238.247:8333 -87.61.90.230:8333 -87.79.68.86:8333 -87.79.94.221:8333 -87.120.8.5:20008 -87.246.46.132:8333 -87.247.111.222:8333 -88.84.222.252:8333 -88.86.243.241:8333 -88.87.93.52:1691 -88.119.197.200:8333 -88.129.253.94:8333 -88.147.244.250:8333 -88.208.3.195:8333 -88.212.44.33:8333 -88.214.57.95:8333 -89.106.199.38:8333 -89.108.126.228:8333 -89.115.120.43:8333 -89.133.68.65:8333 -89.190.19.162:8333 -89.248.172.10:8333 -90.146.153.21:8333 -90.182.165.18:8333 -91.106.188.229:8333 -91.193.237.116:8333 -91.204.99.178:8333 -91.204.149.5:8333 -91.214.70.63:8333 -91.228.152.236:8333 -92.12.154.115:8333 -92.249.143.44:8333 -93.12.66.98:8333 -93.46.54.4:8333 -93.115.20.130:8333 -93.123.180.164:8333 -93.189.145.169:8333 -93.241.228.102:8333 -94.19.7.55:8333 -94.19.128.204:8333 -94.52.112.227:8333 -94.154.96.130:8333 -94.156.174.201:8333 -94.158.246.183:8333 -94.177.171.73:8333 -94.199.178.233:8100 -94.237.125.30:8333 -94.247.134.77:8333 -95.48.228.45:8333 -95.69.249.63:8333 -95.82.146.70:8333 -95.83.73.31:8333 -95.84.164.43:8333 -95.87.226.56:8333 -95.110.234.93:8333 -95.163.71.126:8333 -95.164.65.194:8333 -95.174.66.211:8333 -95.211.174.137:8333 -95.216.11.156:8433 -96.47.114.108:8333 -97.84.232.105:8333 -97.99.205.241:8333 -98.25.193.114:8333 -99.115.25.13:8333 -101.32.19.184:8333 -101.100.174.240:8333 -102.132.245.16:8333 -103.14.244.190:8333 -103.76.48.5:8333 -103.84.84.250:8335 -103.99.168.150:8333 -103.109.101.216:8333 -103.122.247.102:8333 -103.129.13.45:8333 -103.198.192.14:20008 -103.224.119.99:8333 -103.231.191.7:8333 -103.235.230.196:8333 -104.171.242.155:8333 -104.238.220.199:8333 -106.163.158.127:8333 -107.150.41.179:8333 -107.159.93.103:8333 -108.183.77.12:8333 -109.9.175.65:8333 -109.99.63.159:8333 -109.110.81.90:8333 -109.123.213.130:8333 -109.134.232.81:8333 -109.169.20.168:8333 -109.199.241.148:8333 -109.229.210.6:8333 -109.236.105.40:8333 -109.248.206.13:8333 -111.42.74.65:8333 -111.90.140.179:8333 -112.215.205.236:8333 -113.52.135.125:8333 -114.23.246.137:8333 -115.47.141.250:8885 -115.70.110.4:8333 -116.34.189.55:8333 -118.103.126.140:28333 -118.189.187.219:8333 -119.3.208.236:8333 -119.8.47.225:8333 -119.17.151.61:8333 -120.25.24.30:8333 -120.241.34.10:8333 -121.98.205.100:8333 -122.112.148.153:8339 -122.116.42.140:8333 -124.217.235.180:8333 -125.236.215.133:8333 -129.13.189.212:8333 -130.185.77.105:8333 -131.188.40.191:8333 -131.193.220.15:8333 -135.23.124.239:8333 -136.33.185.32:8333 -136.56.170.96:8333 -137.226.34.46:8333 -138.229.26.42:8333 -139.9.249.234:8333 -141.101.8.36:8333 -143.176.224.104:8333 -144.2.69.224:8333 -144.34.161.65:18333 -144.91.116.44:8333 -144.137.29.181:8333 -148.66.50.50:8335 -148.72.150.231:8333 -148.170.212.44:8333 -149.167.99.190:8333 -154.92.16.191:8333 -154.221.27.21:8333 -156.19.19.90:8333 -156.241.5.190:8333 -157.13.61.76:8333 -157.13.61.80:8333 -157.230.166.98:14391 -158.75.203.2:8333 -158.181.125.150:8333 -158.181.226.33:8333 -159.100.242.254:8333 -159.100.248.234:8333 -159.138.87.18:8333 -160.16.0.30:8333 -162.0.227.54:8333 -162.0.227.56:8333 -162.62.18.226:8333 -162.209.1.233:8333 -162.243.175.86:8333 -162.244.80.208:8333 -162.250.188.87:8333 -162.250.189.53:8333 -163.158.202.112:8333 -163.158.243.230:8333 -165.73.62.31:8333 -166.62.82.103:32771 -166.70.94.106:8333 -167.86.90.239:8333 -169.44.34.203:8333 -172.93.101.73:8333 -172.105.7.47:8333 -173.23.103.30:8000 -173.53.79.6:8333 -173.70.12.86:8333 -173.89.28.137:8333 -173.176.184.54:8333 -173.208.128.10:8333 -173.254.204.69:8333 -173.255.204.124:8333 -174.94.155.224:8333 -174.114.102.41:8333 -174.114.124.12:8333 -176.10.227.59:8333 -176.31.224.214:8333 -176.74.136.237:8333 -176.99.2.207:8333 -176.106.191.2:8333 -176.160.228.9:8333 -176.191.182.3:8333 -176.212.185.153:8333 -176.241.137.183:8333 -177.38.215.73:8333 -178.16.222.146:8333 -178.132.2.246:8333 -178.143.191.171:8333 -178.148.172.209:8333 -178.148.226.180:8333 -178.150.96.46:8333 -178.182.227.50:8333 -178.236.137.63:8333 -178.255.42.126:8333 -180.150.52.37:8333 -181.39.32.99:8333 -181.48.77.26:8333 -181.52.223.52:8333 -181.238.51.152:8333 -183.88.223.208:8333 -183.110.220.210:30301 -184.95.58.166:8336 -184.164.147.82:41333 -184.171.208.109:8333 -185.25.48.39:8333 -185.25.48.184:8333 -185.64.116.15:8333 -185.80.219.132:8333 -185.85.3.140:8333 -185.95.219.53:8333 -185.108.244.41:8333 -185.134.233.121:8333 -185.145.128.21:8333 -185.148.3.227:8333 -185.153.196.240:8333 -185.158.114.184:8333 -185.165.168.196:8333 -185.181.230.74:8333 -185.185.26.141:8111 -185.186.208.162:8333 -185.189.132.178:57780 -185.211.59.50:8333 -185.233.148.146:8333 -185.238.129.113:8333 -185.249.199.106:8333 -185.251.161.54:8333 -187.189.153.136:8333 -188.37.24.190:8333 -188.42.40.234:18333 -188.61.46.36:8333 -188.68.45.143:8333 -188.127.229.105:8333 -188.134.6.84:8333 -188.134.8.36:8333 -188.214.129.65:20012 -188.230.168.114:8333 -189.34.14.93:8333 -189.207.46.32:8333 -190.211.204.68:8333 -191.209.21.188:8333 -192.3.11.20:8333 -192.3.185.210:8333 -192.65.170.15:8333 -192.65.170.50:8333 -192.146.137.18:8333 -192.157.202.178:8333 -192.227.80.83:8333 -193.10.203.23:8334 -193.25.6.206:8333 -193.42.110.30:8333 -193.58.196.212:8333 -193.106.28.8:8333 -193.189.190.123:8333 -193.194.163.35:8333 -193.194.163.53:8333 -194.14.246.205:8333 -194.36.91.253:8333 -194.126.113.135:8333 -194.135.135.69:8333 -195.56.63.4:8333 -195.56.63.5:8333 -195.67.139.54:8333 -195.135.194.8:8333 -195.202.169.149:8333 -195.206.105.42:8333 -195.209.249.164:8333 -198.1.231.6:8333 -198.200.43.215:8333 -199.182.184.204:8333 -199.247.7.208:8333 -199.247.249.188:8333 -200.7.252.118:8333 -200.20.186.254:8333 -200.83.166.136:8333 -202.55.87.45:8333 -202.79.167.65:8333 -202.108.211.135:8333 -202.169.102.73:8333 -203.130.48.117:8885 -203.132.95.10:8333 -203.151.166.123:8333 -204.93.113.108:8333 -204.111.241.195:8333 -206.124.149.66:8333 -207.115.102.98:8333 -207.229.46.150:8333 -208.76.252.198:8333 -208.100.13.56:8333 -208.100.178.175:8333 -208.110.99.105:8333 -209.6.210.179:8333 -209.133.220.74:8333 -209.141.57.57:8333 -211.27.147.67:8333 -212.34.225.118:8333 -212.89.173.216:8333 -212.99.226.36:9020 -212.237.96.98:8333 -213.89.131.53:8333 -216.38.129.164:8333 -216.134.165.55:8333 -216.146.251.8:8333 -216.189.190.95:8333 -216.226.128.189:8333 -216.236.164.82:8333 -217.19.216.210:8333 -217.26.32.10:8333 -217.64.47.138:8333 -217.64.133.220:8333 -217.92.55.246:8333 -218.31.113.245:8333 -218.255.242.114:8333 -220.133.39.61:8333 -223.16.30.175:8333 -[2001:19f0:6001:306f:ec4:7aff:fe8f:66ec]:8333 -[2001:1bc0:cc::a001]:8333 -[2001:1c02:2f18:d00:b62e:99ff:fe49:d492]:8333 -[2001:4100:0:64::93]:8333 -[2001:4100:0:64:dcaf:afff:fe00:6707]:8333 -[2001:470:a:c13::2]:8333 -[2001:4801:7819:74:b745:b9d5:ff10:a61a]:8333 -[2001:4ba0:fffa:5d::93]:8333 -[2001:610:1908:ff01:f816:3eff:fe33:2e32]:8333 -[2001:638:a000:4140::ffff:191]:8333 -[2001:648:2800:131:4b1f:f6fc:20f7:f99f]:8333 -[2001:678:7dc:8::2]:8333 -[2001:678:cc8::1:10:88]:20008 -[2001:67c:1220:80c::93e5:dd2]:8333 -[2001:67c:1220:80c:e5dc:ad0c:9289:c28f]:8333 -[2001:67c:16dc:1201:5054:ff:fe17:4dac]:8333 -[2001:67c:2354:2::22]:8333 -[2001:67c:26b4:12:7ae3:b5ff:fe04:6f9c]:8333 -[2001:67c:2f0::20:fa]:8333 -[2001:718:801:311:5054:ff:fe19:c483]:8333 -[2001:8d8:87c:7c00::99:3c1]:8333 -[2001:8f1:1404:3700:8e49:715a:2e09:b634]:9444 -[2001:b07:5d29:99a5:194b:3874:d65e:a90d]:8333 -[2001:ba8:1f1:f0fe::2]:8333 -[2001:bc8:1200:0:dac4:97ff:fe2a:3554]:20008 -[2001:da8:100d:22:10fa:d85f:10f2:21fd]:8333 -[2001:da8:8001:7a39:f035:7d:b99f:eb79]:8333 -[2001:e42:103:100::30]:8333 -[2400:2412:103:c900:825:8f20:eaff:65c2]:8333 -[2400:4052:e20:4f00:69fe:bb33:7b1c:a1ca]:8333 -[2401:1800:7800:105:be76:4eff:fe1c:b35]:8333 -[2401:3900:2:1::2]:8333 -[2401:b140::44:150]:8333 -[2401:d002:4402:0:8f28:591a:6ea0:c683]:8333 -[2403:6200:8821:3d68:195b:87e9:6819:d5c8]:8333 -[2405:6580:2140:3a00:c28c:983:364b:5d70]:8333 -[2405:9800:b911:a18a:58eb:cd3c:9d82:ea4a]:8333 -[2405:aa00:2::40]:8333 -[2409:10:ca20:1df0:224:e8ff:fe1f:60d9]:8333 -[2409:8a1e:a9af:3660:1c5a:5b6b:8a2d:9848]:8333 -[2409:8a1e:a9af:3660:404:39ba:88f2:e8df]:8333 -[240b:10:9141:400:49b4:3a2e:1e5:84c]:8333 -[240d:1a:759:6000:a7b1:451a:8874:e1ac]:8333 -[240d:1a:759:6000:ddab:3141:4da0:8878]:8333 -[2600:8805:2400:14e:12dd:b1ff:fef2:3013]:8333 -[2601:602:8d80:b63:dc3e:24ff:fe92:5eb]:8333 -[2602:ffb6:4:2798:f816:3eff:fe2f:5441]:8333 -[2602:ffb6:4:739e:f816:3eff:fe00:c2b3]:8333 -[2602:ffb8::208:72:57:200]:8333 -[2604:1380:4111:9300::1]:8333 -[2604:4300:a:2e:21b:21ff:fe11:392]:8333 -[2604:4500::2e06]:8112 -[2604:5500:706a:4000:fc79:b9bb:1d7:c325]:8333 -[2604:5500:c134:4000::3fc]:32797 -[2604:6800:5e11:162:5c8f:d2ff:fe26:146f]:8333 -[2605:4d00::50]:8333 -[2605:6400:20:13bf:df1d:181c:83bb:22e8]:8333 -[2605:ae00:203::203]:8333 -[2605:c000:2a0a:1::102]:8333 -[2607:f2c0:f00e:300::54]:8333 -[2607:f2f8:ad40:bc1::1]:8333 -[2607:f470:8:1048:ae1f:6bff:fe70:7240]:8333 -[2607:ff28:800f:97:225:90ff:fe75:1110]:8333 -[2620:11c:5001:1118:d267:e5ff:fee9:e673]:8333 -[2620:6e:a000:2001::6]:8333 -[2804:14d:4c93:9809:9769:da80:1832:3480]:8333 -[2a00:1328:e101:c00::163]:8333 -[2a00:1398:4:2a03:215:5dff:fed6:1033]:8333 -[2a00:13a0:3015:1:85:14:79:26]:8333 -[2a00:1630:14::101]:8333 -[2a00:1768:2001:27::ef6a]:8333 -[2a00:1828:a004:2::666]:8333 -[2a00:1838:36:17::38cb]:8333 -[2a00:1838:36:7d::d3c6]:8333 -[2a00:1c10:2:709:58f7:e0ff:fe24:a0ba]:22220 -[2a00:1c10:2:709::217]:22220 -[2a00:1f40:5001:100::31]:8333 -[2a00:6020:1395:1400:baf7:2d43:60b3:198b]:8333 -[2a00:7c80:0:10b::3faf]:8333 -[2a00:8a60:e012:a00::21]:8333 -[2a00:ab00:603:84::3]:8333 -[2a00:bbe0:cc:0:62a4:4cff:fe23:7510]:8333 -[2a00:ca8:a1f:3025:f949:e442:c940:13e8]:8333 -[2a00:d2a0:a:3d00:1cdf:38bb:a7d6:c251]:8333 -[2a00:d880:11::20e]:8333 -[2a00:ec0:7207:9100:5f8f:25dd:2574:3982]:8333 -[2a00:f820:433::36]:8333 -[2a01:138:a017:b018::42]:8333 -[2a01:430:17:1::ffff:1153]:8333 -[2a01:490:16:301::2]:8333 -[2a01:4b00:807c:1b00:cda1:c6a:2bad:2418]:8333 -[2a01:4b00:80e7:5405::1]:8333 -[2a01:4f8:192:4212::2]:8433 -[2a01:7a0:2:137c::3]:8333 -[2a01:7a7:2:1467:ec4:7aff:fee2:5690]:8333 -[2a01:7c8:d002:10f:5054:ff:fe5c:dac7]:8333 -[2a01:7c8:d002:318:5054:ff:febe:cbb1]:8333 -[2a01:8740:1:ffc5::8c6a]:8333 -[2a01:cb00:f98:ca00:5054:ff:fed4:763d]:8333 -[2a01:cb14:cf6:bc00:21e5:f12e:32c8:145]:8333 -[2a01:d0:0:1c::245]:8333 -[2a01:d0:bef2::12]:8333 -[2a01:e35:2e40:6830:211:32ff:fea6:de3d]:8333 -[2a02:1205:c6aa:60c0:70d8:aaee:a82d:993c]:8333 -[2a02:169:502::614]:8333 -[2a02:180:1:1::5b8f:538c]:8333 -[2a02:348:62:5ef7::1]:8333 -[2a02:390:9000:0:218:7dff:fe10:be33]:8333 -[2a02:7aa0:1619::adc:8de0]:8333 -[2a02:7b40:b0df:8925::1]:8333 -[2a02:7b40:b905:37db::1]:8333 -[2a02:810d:8cbf:f3a8:96c6:91ff:fe17:ae1d]:8333 -[2a02:8389:1c0:9680:201:2eff:fe82:b3cc]:8333 -[2a02:a454:a516:1:517:928:7e0d:957c]:8333 -[2a02:af8:fab0:804:151:236:34:161]:8333 -[2a02:af8:fab0:808:85:234:145:132]:8333 -[2a02:e00:fff0:1e2::a]:8333 -[2a03:2260:3006:d:d307:5d1d:32ca:1fe8]:8333 -[2a03:6000:870:0:46:23:87:218]:8333 -[2a03:9da0:f6:1::2]:8333 -[2a03:c980:db:47::]:8333 -[2a03:e2c0:1ce::2]:8333 -[2a04:3544:1000:1510:706c:abff:fe6c:501c]:8333 -[2a04:52c0:101:383::2a87]:8333 -[2a04:52c0:101:3fb::4c27]:8333 -[2a04:ee41:83:50df:d908:f71d:2a86:b337]:8333 -[2a05:6d40:b94e:d100:225:90ff:fe0d:cfc2]:8333 -[2a05:e5c0:0:100:250:56ff:feb9:d6cb]:8333 -[2a05:fc87:1:6::2]:8333 -[2a05:fc87:4::8]:8333 -[2a07:5741:0:115d::1]:8333 -[2a07:a880:4601:1062:b4b4:bd2a:39d4:7acf]:51401 -[2a07:abc4::1:946]:8333 -[2a07:b400:1:34c::2:1002]:8333 -[2a0a:8c41::b4]:8333 -[2a0a:c801:1:7::183]:8333 -[2a0b:ae40:3:4a0a::15]:8333 -[2a0f:df00:0:254::46]:8333 -[2c0f:f598:5:1:1001::1]:8333 -[2c0f:fce8:0:400:b7c::1]:8333 +2.3.25.181:8333 # AS3215 +2.152.78.124:8333 # AS12430 +5.39.74.166:8333 # AS16276 +5.45.79.81:18332 # AS50673 +5.53.16.128:8333 # AS50923 +5.95.186.78:8333 # AS30722 +5.128.87.126:8333 # AS31200 +5.133.65.82:8333 # AS15440 +5.146.20.229:8333 # AS3209 +5.180.41.119:8333 # AS18978 +5.188.62.18:8333 # AS34665 +5.199.173.66:8333 # AS16125 +5.255.97.25:8333 # AS60404 +5.255.103.180:8333 # AS60404 +8.209.70.77:8333 # AS45102 +8.209.105.138:8333 # AS45102 +18.162.208.153:48332 # AS16509 +23.175.0.200:8333 # AS395502 +23.175.0.222:8333 # AS395502 +23.233.107.21:8333 # AS5645 +23.236.25.169:8333 # AS30029 +24.35.68.229:8333 # AS11404 +24.84.164.50:8333 # AS6327 +24.116.153.115:8333 # AS11492 +24.184.0.146:8333 # AS6128 +27.33.160.196:8333 # AS7545 +27.124.108.19:8333 # AS58511 +27.148.206.140:8333 # AS4134 +31.17.64.192:8333 # AS204028 +31.18.114.135:8333 # AS204028 +31.41.23.249:8333 # AS31287 +31.42.176.138:8333 # AS43641 +31.47.202.112:8333 # AS34385 +34.65.45.157:8333 # AS15169 +34.80.134.68:8333 # AS15169 +34.126.115.35:8333 # AS396982 +37.1.204.231:8333 # AS50673 +37.120.155.34:8333 # AS9009 +37.143.118.174:8333 # AS48926 +37.193.227.16:8333 # AS31200 +37.220.135.151:8333 # AS41206 +37.235.146.236:8333 # AS41268 +38.124.126.42:8333 # AS11550 +38.141.134.140:8333 # AS174 +38.145.151.150:8333 # AS40545 +40.115.137.28:8333 # AS8075 +41.72.154.66:8333 # AS37153 +41.79.70.146:8333 # AS37349 +42.193.55.135:8333 # AS45090 +43.225.62.107:8333 # AS63953 +45.43.97.103:8333 # AS26827 +45.85.48.58:8333 # AS208016 +45.126.26.229:8333 # AS45763 +45.134.142.40:8333 # AS60068 +45.154.252.162:8333 # AS13335 +46.13.216.169:8333 # AS6855 +46.23.87.218:8333 # AS51088 +46.40.127.164:8333 # AS43205 +46.48.126.58:8333 # AS12668 +46.59.13.35:8333 # AS8473 +46.72.238.17:8333 # AS12714 +46.128.141.184:8333 # AS16097 +46.146.248.89:8333 # AS9049 +46.165.221.209:9333 # AS28753 +46.166.142.2:8333 # AS43350 +46.175.178.3:8333 # AS28725 +47.36.144.51:8333 # AS20115 +47.180.49.158:8333 # AS5650 +49.228.131.133:2210 # AS133481 +50.2.13.164:8333 # AS62904 +50.35.71.51:8333 # AS20055 +50.53.250.162:8333 # AS20055 +51.68.36.57:8333 # AS16276 +51.138.4.135:30001 # AS8075 +51.154.62.103:8333 # AS15796 +51.158.150.155:8333 # AS12876 +54.176.63.16:8333 # AS16509 +58.158.0.86:8333 # AS2519 +59.138.115.137:8333 # AS2516 +59.167.191.60:8333 # AS4739 +60.205.205.119:8333 # AS37963 +60.234.122.245:8333 # AS9790 +60.240.210.155:8333 # AS7545 +61.239.91.250:8333 # AS9269 +62.74.143.11:8333 # AS3329 +62.138.162.12:8333 # AS20773 +62.169.74.233:8333 # AS2860 +62.171.129.32:8333 # AS51167 +62.209.198.65:8333 # AS6855 +63.247.147.166:8333 # AS30221 +64.98.76.62:8333 # AS32133 +66.29.129.218:8333 # AS22612 +66.96.235.28:8333 # AS63859 +66.130.120.52:8333 # AS5769 +66.198.209.243:8333 # AS33152 +66.208.64.128:8333 # AS10352 +66.225.231.148:8333 # AS23352 +67.55.3.200:8333 # AS33139 +67.58.232.107:8333 # AS14051 +67.211.92.2:8333 # AS11711 +67.223.119.122:8333 # AS22612 +68.48.131.251:8333 # AS7922 +68.181.4.12:8333 # AS47 +69.14.185.9:8333 # AS12083 +69.54.29.193:8333 # AS12282 +69.59.18.22:8333 # AS397444 +69.131.101.176:8333 # AS4181 +69.165.205.142:8833 # AS5645 +69.228.219.124:8333 # AS7018 +70.59.123.25:8333 # AS209 +70.62.13.150:8333 # AS7843 +70.66.248.170:8333 # AS6327 +70.112.153.229:8333 # AS7843 +70.160.240.132:8333 # AS22773 +70.190.177.204:8333 # AS22773 +71.28.189.239:8333 # AS398465 +71.234.125.198:8333 # AS1351 +72.74.123.179:8333 # AS701 +72.253.236.217:8333 # AS36149 +73.219.254.120:8333 # AS1351 +74.91.115.229:8333 # AS14586 +74.118.137.119:8333 # AS20326 +74.195.166.100:8333 # AS19108 +74.220.255.190:8333 # AS23175 +76.67.211.110:8333 # AS577 +76.169.163.14:8333 # AS20001 +77.32.121.162:8333 # AS35612 +77.53.135.74:8333 # AS45011 +77.70.16.245:8333 # AS8717 +77.85.204.149:8333 # AS8866 +77.107.38.239:8333 # AS62183 +77.120.26.102:8333 # AS25229 +77.162.190.90:8333 # AS1136 +78.20.227.249:8333 # AS6848 +78.21.167.8:8333 # AS6848 +78.27.139.13:8333 # AS6723 +78.90.91.220:8333 # AS8717 +78.108.108.25:8333 # AS8251 +78.108.108.38:8333 # AS8251 +79.77.182.183:8333 # AS13285 +79.98.159.7:11333 # AS44065 +79.189.211.201:8333 # AS5617 +80.55.225.158:8333 # AS5617 +80.83.186.35:8333 # AS33891 +80.88.172.227:64264 # AS31263 +80.209.87.103:9333 # AS31027 +80.229.28.60:8333 # AS2856 +81.7.16.182:8333 # AS35366 +81.7.17.202:8333 # AS35366 +81.19.10.2:8333 # AS24641 +81.88.221.190:8333 # AS39709 +81.171.22.143:8333 # AS60781 +81.224.44.164:8333 # AS3301 +81.224.160.81:8333 # AS3301 +82.1.68.54:8333 # AS5089 +82.21.164.47:8333 # AS5089 +82.64.116.5:8333 # AS12322 +82.66.10.11:8333 # AS12322 +82.96.96.40:8333 # AS29686 +82.116.50.101:8333 # AS30936 +82.129.68.62:8333 # AS48945 +82.136.99.122:8333 # AS8821 +82.154.24.209:8333 # AS8657 +82.197.215.125:8333 # AS25596 +83.128.132.91:8333 # AS15435 +83.137.41.10:8333 # AS31394 +83.208.6.211:8333 # AS5610 +83.208.193.242:8333 # AS5610 +83.222.138.85:8333 # AS31736 +83.240.124.68:8333 # AS31246 +83.243.191.199:8333 # AS41164 +84.9.5.211:8333 # AS5378 +84.28.57.90:8333 # AS6830 +84.38.3.249:8333 # AS196691 +84.112.60.16:8333 # AS8412 +84.215.56.119:8333 # AS41164 +84.226.243.175:8333 # AS6730 +84.245.14.73:8333 # AS25596 +84.252.157.90:18333 # AS200590 +84.255.244.61:8333 # AS34779 +85.23.24.123:8333 # AS16086 +85.52.185.29:8666 # AS12479 +85.58.120.201:8333 # AS12479 +85.93.96.18:8333 # AS29208 +85.165.8.197:8333 # AS2119 +85.173.165.66:8333 # AS12389 +85.184.143.105:8333 # AS39642 +85.191.74.103:8333 # AS39642 +85.194.238.134:8333 # AS47605 +85.195.54.110:8333 # AS35706 +85.195.196.142:8333 # AS13030 +85.208.69.11:8333 # AS25091 +85.208.69.21:8333 # AS25091 +85.208.71.36:8333 # AS42275 +85.208.71.39:8333 # AS42275 +85.214.118.71:8333 # AS6724 +85.214.161.252:8333 # AS6724 +85.216.32.73:8333 # AS51185 +85.254.98.221:8333 # AS13194 +86.58.11.152:8333 # AS3212 +86.95.8.249:8333 # AS1136 +86.100.26.188:8333 # AS39007 +86.106.143.143:55373 # AS9009 +86.124.145.184:8333 # AS8708 +86.133.251.239:8901 # AS2856 +87.79.94.221:8333 # AS8422 +87.120.8.5:20008 # AS34224 +87.125.157.220:8333 # AS12430 +88.9.76.133:8333 # AS3352 +88.90.184.68:8333 # AS2119 +88.151.101.14:5000 # AS41075 +88.151.101.253:5000 # AS41075 +88.198.92.47:8333 # AS24940 +88.208.115.70:8333 # AS29208 +88.210.15.24:8333 # AS212702 +88.212.45.166:8333 # AS42841 +89.102.206.238:8333 # AS16019 +89.103.111.34:8333 # AS16019 +89.114.143.113:8333 # AS12353 +89.134.62.74:8333 # AS21334 +89.152.8.231:8333 # AS2860 +89.161.26.78:8333 # AS39375 +89.207.131.19:8333 # AS49544 +89.248.193.229:8333 # AS49505 +90.3.48.62:8333 # AS3215 +90.146.121.97:8333 # AS12605 +90.146.130.214:8333 # AS12605 +90.196.169.58:8333 # AS5607 +90.250.9.1:8333 # AS5378 +91.93.194.154:8333 # AS34984 +91.126.40.109:8333 # AS35699 +91.204.99.178:8333 # AS20485 +91.204.149.5:8333 # AS42765 +91.206.17.195:8333 # AS13259 +91.209.51.131:8333 # AS48239 +91.215.91.254:8333 # AS48078 +92.91.27.60:8333 # AS15557 +92.221.20.232:8333 # AS29695 +92.255.85.31:8333 # AS9002 +93.4.101.37:8333 # AS15557 +93.46.81.5:8333 # AS12874 +93.57.81.162:8333 # AS12874 +93.73.39.196:8333 # AS25229 +93.90.82.226:8333 # AS47626 +93.95.88.13:8333 # AS35434 +93.123.180.164:8333 # AS35539 +93.189.145.169:8333 # AS12555 +94.17.185.107:8333 # AS12709 +94.75.198.120:8333 # AS60781 +94.114.196.169:8333 # AS3209 +94.142.213.250:55544 # AS5524 +94.154.159.99:8333 # AS62240 +94.158.246.183:8333 # AS39798 +94.239.145.32:8333 # AS5410 +95.31.12.22:8333 # AS8402 +95.31.196.15:8333 # AS3216 +95.110.133.223:8333 # AS31034 +95.110.234.93:8333 # AS31034 +95.161.12.45:8333 # AS39598 +95.191.130.100:8333 # AS12389 +95.208.158.161:8333 # AS51185 +95.213.145.218:8333 # AS49505 +95.214.53.154:8333 # AS201814 +95.214.53.160:8333 # AS201814 +96.44.156.199:8333 # AS8100 +97.75.145.12:8333 # AS22709 +102.132.192.141:8333 # AS37680 +103.14.245.250:8333 # AS24482 +103.85.38.205:8333 # AS134090 +103.88.92.78:8332 # AS17547 +103.99.168.100:8333 # AS6939 +103.99.168.140:8333 # AS6939 +103.99.170.210:8333 # AS54415 +103.99.170.220:8333 # AS54415 +103.100.44.70:8333 # AS10143 +103.178.236.27:8333 # AS49981 +103.209.12.144:8333 # AS58511 +104.59.147.15:8333 # AS7018 +104.129.171.121:8333 # AS174 +104.200.65.234:8333 # AS23033 +104.238.220.199:8333 # AS23470 +104.244.73.6:8333 # AS53667 +106.71.119.230:8333 # AS4804 +107.173.166.43:8333 # AS23352 +108.161.22.78:8333 # AS54154 +108.174.63.234:8333 # AS36352 +109.99.63.159:8333 # AS9050 +109.105.40.247:8333 # AS12570 +109.107.185.130:8333 # AS48282 +109.110.239.4:8333 # AS35432 +109.173.41.43:8333 # AS42610 +109.236.90.117:8333 # AS49981 +109.248.206.13:8333 # AS203493 +109.255.106.206:8333 # AS6830 +111.90.140.23:8333 # AS45839 +111.90.140.46:8333 # AS45839 +111.90.159.246:8333 # AS34309 +112.118.188.50:8333 # AS4760 +115.47.141.250:8885 # AS4134 +116.58.171.67:8333 # AS2514 +118.92.107.108:8333 # AS9500 +119.42.55.203:8333 # AS133159 +120.79.71.72:8333 # AS37963 +121.99.240.87:8333 # AS9790 +123.60.213.192:8333 # AS55990 +124.156.158.100:8333 # AS132203 +124.222.123.238:8333 # AS45090 +125.178.6.116:8333 # AS3786 +128.0.190.26:8333 # AS30764 +128.65.194.136:8333 # AS29222 +129.13.189.212:8333 # AS34878 +129.126.172.115:8333 # AS17547 +129.146.52.174:8333 # AS31898 +130.44.168.202:8333 # AS6079 +131.161.80.166:8333 # AS263694 +131.188.40.191:8333 # AS680 +134.195.185.52:8333 # AS13536 +135.134.238.47:8333 # AS4181 +135.180.218.58:8333 # AS46375 +135.181.215.237:8333 # AS24940 +136.29.109.180:8333 # AS19165 +136.32.238.6:8333 # AS16591 +136.56.170.96:8333 # AS16591 +137.25.38.108:8333 # AS20115 +137.226.34.46:8333 # AS680 +138.207.211.106:8333 # AS11776 +139.130.41.82:8333 # AS1221 +139.153.255.107:8333 # AS786 +140.190.12.129:8333 # AS14828 +142.54.181.218:8333 # AS32097 +143.177.229.149:8333 # AS50266 +143.178.64.10:8333 # AS50266 +144.24.245.183:8333 # AS31898 +144.126.130.178:8333 # AS40021 +146.4.124.129:8333 # AS3303 +146.71.69.103:8333 # AS7782 +146.83.56.69:8333 # AS23140 +147.194.177.165:8333 # AS15128 +149.90.214.78:8333 # AS12353 +149.102.157.156:8333 # AS13768 +151.248.156.55:8333 # AS8821 +151.252.193.245:8333 # AS29582 +153.92.93.114:8333 # AS41998 +154.211.6.2:8333 # AS140224 +156.17.103.2:8088 # AS8970 +156.146.177.221:8333 # AS1448 +157.131.143.173:8333 # AS46375 +158.58.188.37:8333 # AS57497 +158.248.39.239:8333 # AS29695 +159.89.230.128:8333 # AS14061 +159.196.3.239:8333 # AS4764 +159.224.189.250:8333 # AS13188 +160.72.51.154:8333 # AS46887 +161.29.236.55:8333 # AS4826 +161.97.119.166:8333 # AS51167 +161.246.11.230:8333 # AS9486 +162.62.18.226:8333 # AS132203 +162.250.123.179:8333 # AS19318 +162.250.191.222:8333 # AS26832 +162.254.118.20:8333 # AS6130 +163.172.81.70:8333 # AS12876 +164.90.47.8:8333 # AS53449 +165.228.174.117:8333 # AS1221 +166.70.145.151:8333 # AS6315 +168.91.238.8:8333 # AS11039 +170.253.11.25:8333 # AS15704 +171.103.170.115:8333 # AS7470 +172.93.166.135:8333 # AS22653 +172.103.217.236:8333 # AS25668 +172.105.21.216:8333 # AS63949 +172.112.153.95:8333 # AS20001 +173.3.218.91:8333 # AS6128 +173.12.119.133:8333 # AS7922 +173.34.127.181:8333 # AS812 +173.76.123.173:8333 # AS701 +173.176.198.68:8333 # AS5769 +173.208.152.218:8333 # AS32097 +173.241.227.243:8333 # AS19009 +173.246.27.7:8333 # AS1403 +173.255.240.205:8333 # AS63949 +174.30.47.15:8333 # AS209 +174.114.250.86:8333 # AS812 +174.138.35.229:8333 # AS14061 +174.142.191.136:8333 # AS32613 +176.10.143.190:8333 # AS8473 +176.74.136.237:8333 # AS35613 +176.118.220.29:8333 # AS60042 +176.126.116.7:8333 # AS20473 +176.126.167.10:8333 # AS8449 +176.212.185.153:8333 # AS9049 +176.235.209.186:8333 # AS34984 +177.81.236.117:8333 # AS28573 +177.89.205.70:8333 # AS28220 +178.48.168.12:8333 # AS21334 +178.124.162.209:8333 # AS6697 +178.159.98.133:8333 # AS202390 +178.196.89.209:8333 # AS3303 +178.236.137.63:8333 # AS44843 +178.252.123.24:8333 # AS42893 +179.43.170.186:8333 # AS51852 +180.150.46.187:8333 # AS4764 +181.117.128.140:8333 # AS19037 +184.19.19.16:8333 # AS5650 +185.21.217.48:8333 # AS200052 +185.25.48.184:8333 # AS61272 +185.31.136.246:8333 # AS47605 +185.52.93.45:8333 # AS39449 +185.64.116.15:8333 # AS31736 +185.68.249.91:8333 # AS51184 +185.98.54.20:8333 # AS39572 +185.107.83.55:8333 # AS43350 +185.140.253.169:8333 # AS200735 +185.148.145.74:8333 # AS44901 +185.165.170.19:8333 # AS3223 +185.167.113.59:8333 # AS207054 +185.185.26.141:8111 # AS201206 +185.197.163.136:8333 # AS60144 +185.209.12.76:8333 # AS212323 +185.209.70.17:8333 # AS204568 +185.227.156.226:8333 # AS209846 +185.233.189.210:8333 # AS61303 +185.239.221.5:8333 # AS61282 +185.244.100.106:8333 # AS2586 +185.254.97.164:8333 # AS44486 +186.33.167.11:8333 # AS1299 +186.176.98.37:8333 # AS262197 +186.249.217.25:8333 # AS7195 +186.250.95.132:8333 # AS262967 +188.32.14.31:8334 # AS42610 +188.35.167.14:8333 # AS34123 +188.68.45.143:8333 # AS47147 +188.117.200.212:8333 # AS25447 +188.138.88.14:8333 # AS20773 +188.151.237.158:8333 # AS1257 +188.154.236.49:8333 # AS6730 +189.123.177.128:8333 # AS4230 +190.123.27.11:8333 # AS52468 +190.145.127.254:8333 # AS14080 +192.69.53.77:8333 # AS11142 +192.146.137.44:8333 # AS25376 +192.222.24.54:8333 # AS22646 +192.222.147.141:8333 # AS1403 +193.32.127.162:60969 # AS39351 +193.111.198.187:8111 # AS24961 +193.196.37.62:8333 # AS34878 +194.13.80.185:15430 # AS47147 +194.147.113.201:8333 # AS21232 +194.165.30.20:8333 # AS35162 +194.191.239.98:8333 # AS1836 +195.56.63.4:8333 # AS5483 +195.56.63.10:8333 # AS5483 +195.123.239.185:8333 # AS64010 +195.140.226.154:8333 # AS35614 +198.1.231.6:8333 # AS30236 +198.148.112.27:8333 # AS35916 +199.126.234.237:8333 # AS395570 +199.193.174.173:8333 # AS7992 +199.247.7.208:8333 # AS20473 +200.122.181.46:8333 # AS3790 +201.191.6.103:8333 # AS11830 +201.212.36.209:8333 # AS7303 +201.221.234.200:8333 # AS27928 +202.108.211.135:8333 # AS4837 +202.169.17.178:8333 # AS137549 +202.177.24.140:8333 # AS7479 +203.130.48.117:8885 # AS54994 +203.132.94.196:8333 # AS38195 +205.178.41.124:8333 # AS11039 +206.72.201.228:8333 # AS19318 +206.192.203.0:8333 # AS7029 +206.223.153.52:8333 # AS19214 +207.134.216.145:8334 # AS395570 +207.188.154.50:8333 # AS15704 +207.229.46.80:8333 # AS852 +207.255.193.47:8333 # AS11776 +208.104.92.74:8333 # AS14615 +209.58.145.157:8333 # AS394380 +209.58.158.232:8335 # AS394380 +209.141.43.243:8333 # AS53667 +209.226.142.62:8333 # AS577 +209.237.127.227:8333 # AS1299 +209.237.133.54:8333 # AS53859 +211.248.90.50:8333 # AS4766 +212.21.18.78:8333 # AS20485 +212.34.225.118:8333 # AS44395 +212.51.146.137:8333 # AS13030 +212.227.211.87:8333 # AS8560 +213.0.69.76:8333 # AS3352 +213.5.36.58:8333 # AS49974 +213.47.64.105:8333 # AS8412 +213.89.135.151:8333 # AS1257 +213.141.154.201:8333 # AS12714 +213.159.198.45:8333 # AS8359 +213.184.244.24:8333 # AS60280 +213.214.66.182:8333 # AS43205 +213.226.123.76:8333 # AS49943 +216.146.251.8:8333 # AS54579 +216.186.238.14:8333 # AS12083 +217.5.150.114:8333 # AS3320 +217.15.178.11:8333 # AS25534 +217.24.239.109:8333 # AS9063 +217.64.47.138:8333 # AS39324 +217.73.80.104:8333 # AS44291 +217.79.181.38:8333 # AS24961 +217.92.55.246:8333 # AS3320 +217.113.121.169:8333 # AS8416 +217.115.116.250:8333 # AS30900 +217.155.244.170:8333 # AS13037 +217.170.124.170:8333 # AS35401 +220.132.135.54:8333 # AS3462 +220.233.178.199:8333 # AS38195 +222.154.111.46:8333 # AS4648 +[2001:1620:510::2]:8333 # AS13030 +[2001:19f0:6001:39aa:5400:3ff:fef0:916]:8333 # AS20473 +[2001:19f0:8001:f71:5400:4ff:fe10:6a63]:8333 # AS20473 +[2001:1bc0:c1::2000]:8333 # AS29686 +[2001:1c02:11e:3500:df25:6321:8260:d9be]:8333 # AS6830 +[2001:41d0:1004:1b79::]:8339 # AS16276 +[2001:41d0:203:3739::]:8333 # AS16276 +[2001:41d0:203:aacc::]:8333 # AS16276 +[2001:41d0:203:bb0a::]:8333 # AS16276 +[2001:41d0:2:bf8f::]:8333 # AS16276 +[2001:41d0:303:6586::]:8333 # AS16276 +[2001:41d0:602:4493::]:8333 # AS16276 +[2001:41d0:8:b9d8::1]:8333 # AS16276 +[2001:41d0:a:69a2::1]:8333 # AS16276 +[2001:41f0::62:6974:636f:696e]:8333 # AS6830 +[2001:44b8:256:5d11:216:3eff:fe39:d5d4]:8333 # AS4739 +[2001:470:1b62::]:8333 # AS6939 +[2001:470:1f07:803:20c:29ff:fe2d:5879]:8333 # AS6939 +[2001:470:1f15:106:e2d5:5eff:fe42:7ae5]:8333 # AS6939 +[2001:470:1f15:c43::11]:8333 # AS6939 +[2001:470:26:472::b7c]:8333 # AS6939 +[2001:470:75e9:1::10]:8333 # AS6939 +[2001:470:de5a::ec]:9333 # AS6939 +[2001:4ba0:babe:584::1]:8333 # AS24961 +[2001:4ba0:ffff:24::1]:8333 # AS24961 +[2001:4dd0:3564:0:30b7:1d7b:6fec:4c5c]:8333 # AS8422 +[2001:4dd0:3564:0:88e:b4ff:2ad0:699b]:8333 # AS8422 +[2001:4dd0:3564:0:9c1c:cc31:9fe8:5505]:8333 # AS8422 +[2001:4dd0:3564:0:a0c4:d41f:4c4:1bb0]:8333 # AS8422 +[2001:4dd0:3564:0:fd76:c1d3:1854:5bd9]:8333 # AS8422 +[2001:4dd0:3564:1::7676:8090]:8333 # AS8422 +[2001:4dd0:3564:1:b977:bd71:4612:8e40]:8333 # AS8422 +[2001:4dd0:af0e:3564::69:1]:8333 # AS8422 +[2001:4dd0:af0e:3564::69:90]:8333 # AS8422 +[2001:4de8:b1b2:1:0:dead:beef:7]:8333 # AS29208 +[2001:638:a000:4140::ffff:191]:8333 # AS680 +[2001:678:acc:42::]:8333 # AS60404 +[2001:67c:26b4:ff00::44]:8333 # AS25376 +[2001:67c:2db8:6::36]:8333 # AS39798 +[2001:7c0:2310:0:f816:3eff:fe0d:4ab6]:8333 # AS34878 +[2001:7c0:2310:0:f816:3eff:fe6c:4f58]:8333 # AS34878 +[2001:861:3246:a10::40]:8333 # AS5410 +[2001:b07:2e6:38d7:ba27:ebff:fe60:3dc1]:8333 # AS12874 +[2001:b07:6461:7811:489:d2da:e07:1af7]:8333 # AS12874 +[2001:b07:ac9:442b:79d6:bbbe:b37c:a783]:8333 # AS12874 +[2001:bc8:1600:0:208:a2ff:fe0c:8a2e]:8333 # AS12876 +[2001:bc8:323c:ff:a634:384f:1849:f4bc]:8333 # AS12876 +[2001:bc8:323c:ff:d217:c2ff:fe07:2cd9]:8333 # AS12876 +[2001:bc8:3bec:100::1]:8333 # AS12876 +[2002:2f5b:a5f9::2f5b:a5f9]:8885 # AS6939 +[2003:cb:8713:6102:aaa1:59ff:fe57:7779]:8333 # AS3320 +[2003:e0:370e:1400::5]:8333 # AS3320 +[2003:f6:3f10:6700:4c9f:7620:8324:d4a7]:8333 # AS3320 +[2400:2410:cea2:d00:41bc:c9ea:861b:51ee]:8333 # AS17676 +[2400:2411:a3e1:4900:2568:684b:e99:7120]:8333 # AS17676 +[2400:2411:a3e1:4900:2987:b88f:61e0:84fa]:8333 # AS17676 +[2400:3b00:20:c:bacb:29ff:feab:8886]:8333 # AS18229 +[2401:b140:1::100:210]:8333 # AS54415 +[2401:b140:1::100:220]:8333 # AS54415 +[2401:b140::42:100]:8333 # AS6939 +[2401:b140::44:130]:8333 # AS6939 +[2401:d002:3902:700:d72c:5e22:4e95:389d]:8333 # AS38195 +[2404:4408:6752:c000::1999]:8333 # AS9790 +[2404:7a85:4161:2b00:49a1:427a:fac:3409]:8333 # AS2518 +[2405:9800:b972:ab58:c05:e938:267e:271]:8333 # AS45430 +[2406:da11:169:b03:32b5:f901:9f7c:3e4b]:8333 # AS16509 +[2406:da14:335:b601:ceb7:b4fc:a855:f3a5]:8333 # AS16509 +[2406:da1e:a4e:8a03:2aad:496b:768d:e497]:8333 # AS16509 +[2407:8800:bc61:2202:a0c6:107:502b:4e3b]:8333 # AS7545 +[2409:10:ca20:1df0:224:e8ff:fe1f:60d9]:8333 # AS55391 +[2600:1700:22f1:641f:e8:39c8:eb1d:a1eb]:8333 # AS7018 +[2600:1700:9c5d:ed0::38]:8333 # AS7018 +[2600:1700:9c5d:ed0:d0d6:1d9:5cc2:ab47]:8333 # AS7018 +[2600:1702:1ce0:4010::40]:8333 # AS7018 +[2600:1f14:40e:e301:d155:aa3a:77be:960e]:8333 # AS16509 +[2600:1f16:a08:b901:1afa:ef4e:4ce7:2ba4]:8333 # AS16509 +[2600:1f1c:2d3:2403:5bac:3fc6:6513:7a63]:8333 # AS16509 +[2600:2104:1003:c5ab:dc5e:90ff:fe18:1d08]:8333 # AS11404 +[2600:3c00::f03c:92ff:fe92:2745]:8333 # AS63949 +[2600:3c00::f03c:92ff:fecf:61b6]:8333 # AS63949 +[2600:3c00::f03c:93ff:feb3:1b6]:8333 # AS63949 +[2600:3c00:e002:2e32::1:14]:8333 # AS63949 +[2600:3c02::f03c:92ff:fe5d:9fb]:8333 # AS63949 +[2600:4040:2854:5e00:c6e9:84ff:fe46:ee8]:8666 # AS13786 +[2600:6c54:7100:1ad1:bddf:550e:91be:f9e1]:8333 # AS20115 +[2600:8805:2400:14e:12dd:b1ff:fef2:3013]:8333 # AS22773 +[2601:184:300:bde:3c29:8e94:1ba8:fde3]:8333 # AS7922 +[2601:18c:8080:300f:219:d1ff:fe75:dc2f]:8333 # AS7922 +[2601:18d:4600:43f1:20e7:b3ff:fecf:a99]:8333 # AS7922 +[2601:18d:8701:c290::3330]:8333 # AS7922 +[2601:246:4d7f:9e28:f321:36ca:7a71:c687]:8333 # AS7922 +[2601:640:c201:960d:86eb:f27d:66a2:f2c1]:8333 # AS7922 +[2602:241:75d1:2b90::7840]:8333 # AS46375 +[2602:ffb8::208:72:57:200]:8333 # AS2914 +[2603:3004:6a1:3800:851f:584d:7aba:affb]:8333 # AS7922 +[2603:3004:6a1:3800::4402]:8333 # AS7922 +[2603:3004:70d:1400:8532:2900:ce6f:acdf]:8333 # AS7922 +[2603:3004:745:900:f0d7:556a:a8c:ced5]:8333 # AS7922 +[2603:6080:c000:5d8a::104f]:8333 # AS7843 +[2603:8000:d100:8991:cc29:ccff:fe42:300c]:8333 # AS7843 +[2603:8080:1f07:6fdd:7de2:d969:78c9:b7ea]:8333 # AS7843 +[2603:8080:7300:531::13ea]:8333 # AS7843 +[2603:80a0:703:40f8::38]:8333 # AS7843 +[2604:180:f3::218]:8333 # AS3842 +[2604:3d08:0:5:d941:4b03:a093:131b]:8333 # AS6327 +[2604:7c00:120:4b::eb24]:8333 # AS174 +[2604:a00:21:3043:bf6a:535e:dfeb:5b7b]:8333 # AS19318 +[2604:a880:400:d0::1ce7:4001]:8333 # AS14061 +[2604:a880:400:d0::1d44:e001]:8333 # AS14061 +[2604:a880:400:d0::261f:6001]:8333 # AS14061 +[2604:a880:400:d1::7e2:e001]:8333 # AS14061 +[2604:a880:4:1d0::14:3000]:8333 # AS14061 +[2604:a880:4:1d0::e5:b000]:8333 # AS14061 +[2605:6400:30:f220::]:8333 # AS53667 +[2605:6f80:0:7:fc1b:ccff:fe8a:d822]:8333 # AS53340 +[2605:a140:2076:8253::1]:8333 # AS40021 +[2605:a140:3007:1287::1]:8333 # AS40021 +[2605:ae00:203::203]:8333 # AS7819 +[2605:c000:2a0a:1::102]:8333 # AS7393 +[2607:1a00:1:d::11:7c4d]:8333 # AS22653 +[2607:5300:203:1214::]:8333 # AS16276 +[2607:9280:b:73b:250:56ff:fe14:25b5]:8333 # AS395502 +[2607:9280:b:73b:250:56ff:fe21:9c2f]:8333 # AS395502 +[2607:9280:b:73b:250:56ff:fe21:bf32]:8333 # AS395502 +[2607:9280:b:73b:250:56ff:fe33:4d1b]:8333 # AS395502 +[2607:9280:b:73b:250:56ff:fe3d:401]:8333 # AS395502 +[2607:f2c0:e1c2:69:12c3:7bff:fe4d:9431]:8333 # AS5645 +[2607:f2c0:e1c2:69:ecb2:6e88:9f33:5057]:8333 # AS5645 +[2620:6:2003:105:2d8:61ff:fe0f:853]:8333 # AS25682 +[2620:6e:a000:1:42:42:42:42]:8333 # AS397444 +[2620:a6:2000:1::3:d570]:8333 # AS27566 +[2620:a6:2000:1::5:162a]:8333 # AS27566 +[2620:a6:2000:1::5:1631]:8333 # AS27566 +[2620:a6:2000:1::c:e634]:8333 # AS27566 +[2800:40:33:8ab:a0e7:b215:fc83:5c31]:8333 # AS16814 +[2800:bf0:149:f4b:f8df:8d7d:801b:e25e]:8333 # AS27947 +[2804:14c:198:80d5:7603:41d1:d3fc:e797]:8333 # AS28573 +[2804:14d:ae81:827b:99a8:1e3f:6db2:29db]:8333 # AS4230 +[2804:d57:5537:4800:3e7c:3fff:fe7b:80aa]:8333 # AS8167 +[2a00:12e0:101:99:20c:29ff:fe29:d03f]:8333 # AS6798 +[2a00:1328:e101:c00::163]:8333 # AS31078 +[2a00:1398:4:2a03:215:5dff:fed6:1033]:8333 # AS34878 +[2a00:1398:4:2a03::bc03]:8333 # AS34878 +[2a00:1630:10:1003:0:b19:b00b:babe]:8333 # AS49544 +[2a00:1768:2001:27::ef6a]:8333 # AS43350 +[2a00:1828:a004:2::666]:8333 # AS34240 +[2a00:1c10:2:709::217]:22220 # AS50300 +[2a00:1f40:5001:108:5d17:7703:b0f5:4133]:8333 # AS42864 +[2a00:23c5:fe80:7301:d6ae:52ff:fed5:56a5]:8333 # AS2856 +[2a00:23c6:5c91:5808:c05a:4dff:fe65:9d69]:8333 # AS2856 +[2a00:6020:1bfa:d400:20c:29ff:fe61:4a4c]:8333 # AS60294 +[2a00:6020:b482:9200:491a:358c:d8f7:1da]:8333 # AS60294 +[2a00:6020:b489:2000:5054:ff:fefc:5ed8]:8333 # AS60294 +[2a00:7c80:0:25::e37a]:8333 # AS49981 +[2a00:7c80:0:71::8]:8333 # AS49981 +[2a00:8a60:e012:a00::21]:8333 # AS680 +[2a00:ae40:240e:3200::3]:8333 # AS50923 +[2a00:bbe0:cc:0:62a4:4cff:fe23:7510]:8333 # AS47605 +[2a00:ca8:a1f:3025:f949:e442:c940:13e8]:8333 # AS30764 +[2a00:d4e0:2:d002:4467:31e0:6fa5:b3ef]:8333 # AS15600 +[2a00:ee2:1200:1900:8d3:d2ff:feb1:bc58]:8333 # AS5603 +[2a01:238:420f:9200:fa5a:1a4b:1e6a:fadf]:8333 # AS6724 +[2a01:238:4389:c400:3b26:d94e:38d5:44ef]:8333 # AS6724 +[2a01:490:16:301::2]:8333 # AS8251 +[2a01:4b00:807c:3100:cda1:c6a:2bad:2418]:8333 # AS56478 +[2a01:4f8:141:2254::2]:8333 # AS24940 +[2a01:4f8:173:230a::2]:8333 # AS24940 +[2a01:4f8:190:91c4::2]:8333 # AS24940 +[2a01:4f8:200:7222::2]:8333 # AS24940 +[2a01:4f8:202:3e6::2]:8333 # AS24940 +[2a01:4f8:221:44d7::2]:8333 # AS24940 +[2a01:4f8:231:915::2]:8333 # AS24940 +[2a01:4f9:2a:1ce0::2]:8333 # AS24940 +[2a01:4f9:2b:29a::2]:8333 # AS24940 +[2a01:4f9:4a:31de::2]:8333 # AS24940 +[2a01:5200:6c:6162:7a61:746b:6f2e:736b]:8333 # AS6855 +[2a01:6380:fffe:73:10fb:d012:8581:b4d7]:8333 # AS25540 +[2a01:7a7:2:2804:ae1f:6bff:fe9d:6c94]:8333 # AS20773 +[2a01:7c8:aaac:89:5054:ff:feb7:f5cb]:8333 # AS20857 +[2a01:7c8:aac9:c9:5054:ff:fedf:ff95]:8333 # AS20857 +[2a01:7c8:d001:1c1:5054:ff:feee:3e1a]:8333 # AS20857 +[2a01:7c8:d009:2aa:5054:ff:fe1b:a196]:11520 # AS20857 +[2a01:7c8:fffa:50e:ddfe:c924:ca0a:cbab]:8333 # AS20857 +[2a01:7e00::f03c:93ff:fe59:66dc]:8333 # AS63949 +[2a01:7e01::f03c:93ff:fe3b:bb5b]:8333 # AS63949 +[2a01:8740:1:ffc5::8c6a]:8333 # AS57344 +[2a01:9f40:a000::100]:8333 # AS42908 +[2a01:cb00:d3d:7700:227:eff:fe28:c565]:8333 # AS3215 +[2a01:e0a:20:7350:919c:b1c3:8b83:adf9]:8333 # AS12322 +[2a01:e0a:301:7010:b87d:e14b:cea9:b998]:8333 # AS12322 +[2a01:e0a:48b:2d10:94f2:4d5c:ca5f:bf49]:8333 # AS12322 +[2a01:e0a:530:a0a0:f465:af5:be1b:9075]:8333 # AS12322 +[2a01:e0a:aa7:c8c0:9679:affa:b6e5:efc7]:8333 # AS12322 +[2a01:e11:100c:70:cbc8:9e31:4b77:1626]:8333 # AS12322 +[2a01:e34:ee78:3060:230:48ff:fe81:f1c6]:8333 # AS12322 +[2a02:1210:14a9:6700:a00:27ff:fe4e:82b6]:8333 # AS3303 +[2a02:1210:4639:f00:10a7:e965:509a:7a4a]:8333 # AS3303 +[2a02:1210:7c92:5100:211:32ff:feae:152d]:8333 # AS3303 +[2a02:1210:86bf:f100:3178:d700:d44d:6bb1]:8333 # AS3303 +[2a02:1210:9487:a200:edc1:93a4:945:9a92]:8333 # AS3303 +[2a02:168:420b:a::20]:8333 # AS13030 +[2a02:168:6328:0:4a21:bff:fe26:38c3]:8333 # AS13030 +[2a02:168:676e:0:e65f:1ff:fe09:3591]:8333 # AS13030 +[2a02:1748:f39f:5872:dead:beef:b1ac:c0fe]:8333 # AS51184 +[2a02:180:1:1::517:10b6]:8333 # AS35366 +[2a02:2168:a379:d100:96de:80ff:fea3:fd00]:8333 # AS42610 +[2a02:2780:9000:70::7]:8333 # AS35434 +[2a02:2780:9000:70::f]:8333 # AS35434 +[2a02:2780::e01a]:8333 # AS35434 +[2a02:2e02:3900:5400:a099:e1ff:feb6:d0e]:8333 # AS12479 +[2a02:2f05:660e:8b00::1]:8333 # AS48571 +[2a02:58:97:7d20::60]:8333 # AS25596 +[2a02:6d40:3073:c01:dea6:32ff:fe44:4b25]:8333 # AS42652 +[2a02:7a01::91:228:45:130]:8333 # AS16019 +[2a02:7b40:5928:89::1]:8333 # AS62282 +[2a02:7b40:c3b5:f583::1]:8333 # AS62282 +[2a02:8308:8087:aa00:9ea8:1b2:ef98:56bf]:8333 # AS16019 +[2a02:842a:1df:8a01:1e1b:dff:fe0b:236d]:8333 # AS15557 +[2a02:a44d:14d6:1:2c0:8ff:fe8f:b3b2]:8333 # AS1136 +[2a02:a45a:94cd:f00d::1]:8333 # AS1136 +[2a02:a45f:3b9d:30::3]:8333 # AS1136 +[2a02:a467:7833:1:7285:c2ff:fe2c:21e9]:8333 # AS1136 +[2a02:aa14:2380:b300:4040:be88:8b01:d38]:8333 # AS6830 +[2a02:c206:2044:9826::1]:8333 # AS51167 +[2a02:c206:2082:1246::1]:8333 # AS51167 +[2a02:c206:3008:2368::1]:8333 # AS51167 +[2a02:c207:0:4971::1]:5332 # AS51167 +[2a02:c207:2014:4199::1]:8333 # AS51167 +[2a02:c207:2024:6115::1]:8333 # AS51167 +[2a02:c207:2026:6682::1]:8333 # AS51167 +[2a02:c207:3002:7468::1]:8333 # AS51167 +[2a02:e98:20:1504::1]:8333 # AS24641 +[2a03:4000:6:416c::43]:8333 # AS47147 +[2a03:4000:6:f814:548b:17ff:fe31:b64a]:8333 # AS47147 +[2a03:6000:870:0:46:23:87:218]:8333 # AS51088 +[2a03:94e0:ffff:185:243:218:0:19]:8333 # AS56655 +[2a03:b0c0:1:e0::397:6001]:8333 # AS14061 +[2a03:b0c0:2:f0::163:3001]:8333 # AS14061 +[2a03:b0c0:2:f0::18a:d001]:8333 # AS14061 +[2a03:b0c0:3:d0::f3e:2001]:8333 # AS14061 +[2a03:e2c0:1347::2]:8333 # AS50113 +[2a03:ec0:0:928::701:701]:8333 # AS199669 +[2a04:52c0:103:c455::1]:8334 # AS60404 +[2a04:52c0:3007:200::2000]:8333 # AS60404 +[2a04:bc40:1dc3:8d::2:1001]:8333 # AS35277 +[2a05:1500:702:0:1c00:40ff:fe00:c]:8333 # AS48635 +[2a05:3580:d101:3700::]:8333 # AS20764 +[2a05:3580:db0b:1600:c489:76ed:313d:b33]:8333 # AS20764 +[2a05:d014:a55:4001:8127:afa7:daf9:d91b]:8333 # AS16509 +[2a05:d014:a55:4001:f6ab:dd5e:4039:b46c]:8333 # AS16509 +[2a05:d014:a55:4003:6523:50a1:152:e88c]:8333 # AS16509 +[2a05:d01a:b7b:3c01:8bf7:ae14:afb3:33ae]:8333 # AS16509 +[2a05:f480:1800:697:5400:2ff:feb6:c36d]:8333 # AS20473 +[2a06:e040:7603:2918:c6ef:464e:9fe5:73ec]:8333 # AS198507 +[2a07:abc4::1:946]:8333 # AS62000 +[2a09:2681:102::210]:8333 # AS61282 +[2a0a:c801:1:7::183]:8333 # AS39798 +[2a0c:5a80:1210:a800:6af7:28ff:fee5:6b3a]:8333 # AS57269 +[2a0d:5600:24:a8e::a91e]:55373 # AS9009 +[2a0d:7c40:3000:b04::2]:8333 # AS54290 +[2a0d:8340:24::2]:8333 # AS50113 +[2a0f:df00:0:2010::162]:8333 # AS41281 +[2a10:3781:16b9:1:fe3f:dbff:fe04:2d4c]:8333 # AS206238 +[2a10:3781:84b:1:b123:6306:943a:f09b]:8333 # AS206238 +[2a10:d200:1:33:a6bf:1ff:fe6a:46a9]:8333 # AS212323 +[2c0f:f4c0:2202:20b0:261c:4ff:fe14:daa0]:8333 # AS327693 +[2c0f:f8f0:da51:0:70c3:eea9:9717:9579]:8333 # AS30844 -# manually added 2021-03 for minimal torv3 bootstrap support -2g5qfdkn2vvcbqhzcyvyiitg4ceukybxklraxjnu7atlhd22gdwywaid.onion:8333 -2jmtxvyup3ijr7u6uvu7ijtnojx4g5wodvaedivbv74w4vzntxbrhvad.onion:8333 -37m62wn7dz3uqpathpc4qfmgrbupachj52nt3jbtbjugpbu54kbud7yd.onion:8333 +# manually updated 2022-08 for minimal torv3 bootstrap support 5g72ppm3krkorsfopcm2bi7wlv4ohhs4u4mlseymasn7g7zhdcyjpfid.onion:8333 -7cgwjuwi5ehvcay4tazy7ya6463bndjk6xzrttw5t3xbpq4p22q6fyid.onion:8333 -7pyrpvqdhmayxggpcyqn5l3m5vqkw3qubnmgwlpya2mdo6x7pih7r7id.onion:8333 b64xcbleqmwgq2u46bh4hegnlrzzvxntyzbmucn3zt7cssm7y4ubv3id.onion:8333 -ejxefzf5fpst4mg2rib7grksvscl7p6fvjp6agzgfc2yglxnjtxc3aid.onion:8333 fjdyxicpm4o42xmedlwl3uvk5gmqdfs5j37wir52327vncjzvtpfv7yd.onion:8333 fpz6r5ppsakkwypjcglz6gcnwt7ytfhxskkfhzu62tnylcknh3eq6pad.onion:8333 -fzhn4uoxfbfss7h7d6ffbn266ca432ekbbzvqtsdd55ylgxn4jucm5qd.onion:8333 gxo5anvfnffnftfy5frkgvplq3rpga2ie3tcblo2vl754fvnhgorn5yd.onion:8333 ifdu5qvbofrt4ekui2iyb3kbcyzcsglazhx2hn4wfskkrx2v24qxriid.onion:8333 itz3oxsihs62muvknc237xabl5f6w6rfznfhbpayrslv2j2ubels47yd.onion:8333 -lrjh6fywjqttmlifuemq3puhvmshxzzyhoqx7uoufali57eypuenzzid.onion:8333 +kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion:8333 m7cbpjolo662uel7rpaid46as2otcj44vvwg3gccodnvaeuwbm3anbyd.onion:8333 -opnyfyeiibe5qo5a3wbxzbb4xdiagc32bbce46owmertdknta5mi7uyd.onion:8333 -owjsdxmzla6d7lrwkbmetywqym5cyswpihciesfl5qdv2vrmwsgy4uqd.onion:8333 -q7kgmd7n7h27ds4fg7wocgniuqb3oe2zxp4nfe4skd5da6wyipibqzqd.onion:8333 +mwmfluek4au6mxxpw6fy7sjhkm65bdfc7izc7lpz3trewfdghyrzsbid.onion:8333 rp7k2go3s5lyj3fnj6zn62ktarlrsft2ohlsxkyd7v3e3idqyptvread.onion:8333 -sys54sv4xv3hn3sdiv3oadmzqpgyhd4u4xphv4xqk64ckvaxzm57a7yd.onion:8333 -tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333 -vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 -xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 -# manually added 2021-05 for minimal i2p bootstrap support -72l3ucjkuscrbiiepoehuwqgknyzgo7zuix5ty4puwrkyhtmnsga.b32.i2p:0 +# manually updated 2022-08 for minimal i2p bootstrap support +255fhcp6ajvftnyo7bwz3an3t4a4brhopm3bamyh2iu5r3gnr2rq.b32.i2p:0 +27yrtht5b5bzom2w5ajb27najuqvuydtzb7bavlak25wkufec5mq.b32.i2p:0 +2el6enckmfyiwbfcwsygkwksovtynzsigmyv3bzyk7j7qqahooua.b32.i2p:0 +3gocb7wc4zvbmmebktet7gujccuux4ifk3kqilnxnj5wpdpqx2hq.b32.i2p:0 +3tns2oov4tnllntotazy6umzkq4fhkco3iu5rnkxtu3pbfzxda7q.b32.i2p:0 +4fcc23wt3hyjk3csfzcdyjz5pcwg5dzhdqgma6bch2qyiakcbboa.b32.i2p:0 +4osyqeknhx5qf3a73jeimexwclmt42cju6xdp7icja4ixxguu2hq.b32.i2p:0 +4umsi4nlmgyp4rckosg4vegd2ysljvid47zu7pqsollkaszcbpqq.b32.i2p:0 +52v6uo6crlrlhzphslyiqblirux6olgsaa45ixih7sq5np4jujaa.b32.i2p:0 +6j2ezegd3e2e2x3o3pox335f5vxfthrrigkdrbgfbdjchm5h4awa.b32.i2p:0 +6n36ljyr55szci5ygidmxqer64qr24f4qmnymnbvgehz7qinxnla.b32.i2p:0 +72yjs6mvlby3ky6mgpvvlemmwq5pfcznrzd34jkhclgrishqdxva.b32.i2p:0 +7r4ri53lby2i3xqbgpw3idvhzeku7ubhftlf72ldqkg5kde6dauq.b32.i2p:0 +a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0 +aovep2pco7v2k4rheofrgytbgk23eg22dczpsjqgqtxcqqvmxk6a.b32.i2p:0 +bddbsmkas3z6fakorbkfjhv77i4hv6rysyjsvrdjukxolfghc23q.b32.i2p:0 +bitcoi656nll5hu6u7ddzrmzysdtwtnzcnrjd4rfdqbeey7dmn5a.b32.i2p:0 +brifkruhlkgrj65hffybrjrjqcgdgqs2r7siizb5b2232nruik3a.b32.i2p:0 c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 -gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p:0 +day3hgxyrtwjslt54sikevbhxxs4qzo7d6vi72ipmscqtq3qmijq.b32.i2p:0 +di2zq6fr3fegf2jdcd7hdwyql4umr462gonsns2nxz5qg5vz4bka.b32.i2p:0 +e55k6wu46rzp4pg5pk5npgbr3zz45bc3ihtzu2xcye5vwnzdy7pq.b32.i2p:0 +eciohu5nq7vsvwjjc52epskuk75d24iccgzmhbzrwonw6lx4gdva.b32.i2p:0 +ejlnngarmhqvune74ko7kk55xtgbz5i5ncs4vmnvjpy3l7y63xaa.b32.i2p:0 +g47cqoppu26pr4n2cfaioqx7lbdi7mea7yqhlrkdz3wjwxjxdh2a.b32.i2p:0 h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 -hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0 -pjs7or2ctvteeo5tu4bwyrtydeuhqhvdprtujn4daxr75jpebjxa.b32.i2p:0 +hhfi4yqkg2twqiwezrfksftjjofbyx3ojkmlnfmcwntgnrjjhkya.b32.i2p:0 +hpiibrflqkbrcshfhmrtwfyeb7mds7a3obzwrgarejevddzamvsq.b32.i2p:0 +i4pyhsfdq4247dunel7paatdaq5gusi2hnybp2yf5wxwdnrgxaqq.b32.i2p:0 +iw6tgpmbdykffceku5da6nzf2bmz66fvp5fpcvemfu3df6aq6pga.b32.i2p:0 +jkfuajo4ayvo2rbv5qdj443q6adqmnormbhsf2f7rlp5t24xomda.b32.i2p:0 +jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0 +liu75cvktv4icbctg72w7nxbk4eibt7wamizfdii4omz7gcke5vq.b32.i2p:0 +ljsquuu3y4xje6l32p32inn6r2y6ull6oocgup6jtjrohrqxbz6a.b32.i2p:0 +lrah7acdsgopybg43shadwwiv6igezaw64i6jb5muqdg7dmhj3la.b32.i2p:0 +lzuu6mjtu7vd55d2biphicihufipoa7vyym6xfnkmmlra3tiziia.b32.i2p:0 +m6bpynxkv2ktwxkg6p2gyudjfhdupb6kuzabeqdnckkdkf4kxjla.b32.i2p:0 +m6v454xd6p3bt5swujgmveklsp7lzbkqlqqfc2p36cjlwv5dbucq.b32.i2p:0 +mlgeizrroynuhpxbzeosajt5u4ddcvynxfmcbm6kwjpaufilxigq.b32.i2p:0 +ofubxr2ir7u2guzjwyrvujicivzmvinwa36nuzlrg7tnsmebal7a.b32.i2p:0 +okfxeoh6itu4f5f43dhbzvkqwfrvm5c66lj6lvjj4q2b35i4pk4q.b32.i2p:0 +oz2ia3flpm3du2tyusulrn7h7e2eo3juzkrmn34bvnrlcrugv7ia.b32.i2p:0 +qd6jlsevsexww3wefpqs7iglxb3f63y4e6ydulfzrvwflpicmdqa.b32.i2p:0 +qddg7myylinn4tw6kdjmmp6fsyetkosnrbp2gsjx77tmkqyqv6ua.b32.i2p:0 +rizfinyses2r3or4iubs5wx66gdy6mpf73w7uobfacm2l5cral3q.b32.i2p:0 +s5hhjtmlg53bko3nwwskas7xgsmeqzy6thtsj5aa64djyrljgqaq.b32.i2p:0 +sedndhv5vpcgdmykyi5st4yqhdxl3hpdtglta4do435wupahhx6q.b32.i2p:0 +tsl4dlpu2id252b6crbdnblruct664se6f2iw35fuqwa3te7wcoq.b32.i2p:0 +tugq6wa2ls2bv27pr2iy3da3k5ow3fzefbcvjcr22uc7w5vmevja.b32.i2p:0 +usztavbib756k5vqggzgkyswoj6mttihjvp3c2pa642t2mb4pvsa.b32.i2p:0 +vgu6llqbyjphml25umd5ztvyxrxuplz2g74fzbx75g3kkaetoyiq.b32.i2p:0 +wjrul5jwwb4vqdmkkrjbmly7osj6amecdpsac5xvaoqrti4nb3ha.b32.i2p:0 +wvktcp7hy4l6immhi5cxyz2dlsbhhvtcmskjemrnqehacnoap23q.b32.i2p:0 wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 +xlqndzjoe5nr2nsxo6xwibh44ghyz4jfqevu62xykvemextpmjbq.b32.i2p:0 +yc4xwin5ujenvcr6ynwkz7lnmmq3nmzxvfguele6ovqqpxgjvonq.b32.i2p:0 +zdoabsg7ugzothyawodjhq54nvlofa746rxfkxpnjzj6nukmha6a.b32.i2p:0 zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 +zysrlpii5ftrzivfcyhdrwpeyyqddbrdefnfu5q6otk5gtugmh2a.b32.i2p:0 + +# manually added 2022-01 for minimal cjdns bootstrap support +[fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa]:8333 +[fcc7:be49:ccd1:dc91:3125:f0da:457d:8ce]:8333 diff --git a/contrib/seeds/nodes_main_manual.txt b/contrib/seeds/nodes_main_manual.txt new file mode 100644 index 000000000000..286448d95dcc --- /dev/null +++ b/contrib/seeds/nodes_main_manual.txt @@ -0,0 +1,78 @@ + +# manually updated 2022-08 for minimal torv3 bootstrap support +5g72ppm3krkorsfopcm2bi7wlv4ohhs4u4mlseymasn7g7zhdcyjpfid.onion:8333 +b64xcbleqmwgq2u46bh4hegnlrzzvxntyzbmucn3zt7cssm7y4ubv3id.onion:8333 +fjdyxicpm4o42xmedlwl3uvk5gmqdfs5j37wir52327vncjzvtpfv7yd.onion:8333 +fpz6r5ppsakkwypjcglz6gcnwt7ytfhxskkfhzu62tnylcknh3eq6pad.onion:8333 +gxo5anvfnffnftfy5frkgvplq3rpga2ie3tcblo2vl754fvnhgorn5yd.onion:8333 +ifdu5qvbofrt4ekui2iyb3kbcyzcsglazhx2hn4wfskkrx2v24qxriid.onion:8333 +itz3oxsihs62muvknc237xabl5f6w6rfznfhbpayrslv2j2ubels47yd.onion:8333 +kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion:8333 +m7cbpjolo662uel7rpaid46as2otcj44vvwg3gccodnvaeuwbm3anbyd.onion:8333 +mwmfluek4au6mxxpw6fy7sjhkm65bdfc7izc7lpz3trewfdghyrzsbid.onion:8333 +rp7k2go3s5lyj3fnj6zn62ktarlrsft2ohlsxkyd7v3e3idqyptvread.onion:8333 + +# manually updated 2022-08 for minimal i2p bootstrap support +255fhcp6ajvftnyo7bwz3an3t4a4brhopm3bamyh2iu5r3gnr2rq.b32.i2p:0 +27yrtht5b5bzom2w5ajb27najuqvuydtzb7bavlak25wkufec5mq.b32.i2p:0 +2el6enckmfyiwbfcwsygkwksovtynzsigmyv3bzyk7j7qqahooua.b32.i2p:0 +3gocb7wc4zvbmmebktet7gujccuux4ifk3kqilnxnj5wpdpqx2hq.b32.i2p:0 +3tns2oov4tnllntotazy6umzkq4fhkco3iu5rnkxtu3pbfzxda7q.b32.i2p:0 +4fcc23wt3hyjk3csfzcdyjz5pcwg5dzhdqgma6bch2qyiakcbboa.b32.i2p:0 +4osyqeknhx5qf3a73jeimexwclmt42cju6xdp7icja4ixxguu2hq.b32.i2p:0 +4umsi4nlmgyp4rckosg4vegd2ysljvid47zu7pqsollkaszcbpqq.b32.i2p:0 +52v6uo6crlrlhzphslyiqblirux6olgsaa45ixih7sq5np4jujaa.b32.i2p:0 +6j2ezegd3e2e2x3o3pox335f5vxfthrrigkdrbgfbdjchm5h4awa.b32.i2p:0 +6n36ljyr55szci5ygidmxqer64qr24f4qmnymnbvgehz7qinxnla.b32.i2p:0 +72yjs6mvlby3ky6mgpvvlemmwq5pfcznrzd34jkhclgrishqdxva.b32.i2p:0 +7r4ri53lby2i3xqbgpw3idvhzeku7ubhftlf72ldqkg5kde6dauq.b32.i2p:0 +a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0 +aovep2pco7v2k4rheofrgytbgk23eg22dczpsjqgqtxcqqvmxk6a.b32.i2p:0 +bddbsmkas3z6fakorbkfjhv77i4hv6rysyjsvrdjukxolfghc23q.b32.i2p:0 +bitcoi656nll5hu6u7ddzrmzysdtwtnzcnrjd4rfdqbeey7dmn5a.b32.i2p:0 +brifkruhlkgrj65hffybrjrjqcgdgqs2r7siizb5b2232nruik3a.b32.i2p:0 +c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 +day3hgxyrtwjslt54sikevbhxxs4qzo7d6vi72ipmscqtq3qmijq.b32.i2p:0 +di2zq6fr3fegf2jdcd7hdwyql4umr462gonsns2nxz5qg5vz4bka.b32.i2p:0 +e55k6wu46rzp4pg5pk5npgbr3zz45bc3ihtzu2xcye5vwnzdy7pq.b32.i2p:0 +eciohu5nq7vsvwjjc52epskuk75d24iccgzmhbzrwonw6lx4gdva.b32.i2p:0 +ejlnngarmhqvune74ko7kk55xtgbz5i5ncs4vmnvjpy3l7y63xaa.b32.i2p:0 +g47cqoppu26pr4n2cfaioqx7lbdi7mea7yqhlrkdz3wjwxjxdh2a.b32.i2p:0 +h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 +hhfi4yqkg2twqiwezrfksftjjofbyx3ojkmlnfmcwntgnrjjhkya.b32.i2p:0 +hpiibrflqkbrcshfhmrtwfyeb7mds7a3obzwrgarejevddzamvsq.b32.i2p:0 +i4pyhsfdq4247dunel7paatdaq5gusi2hnybp2yf5wxwdnrgxaqq.b32.i2p:0 +iw6tgpmbdykffceku5da6nzf2bmz66fvp5fpcvemfu3df6aq6pga.b32.i2p:0 +jkfuajo4ayvo2rbv5qdj443q6adqmnormbhsf2f7rlp5t24xomda.b32.i2p:0 +jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0 +liu75cvktv4icbctg72w7nxbk4eibt7wamizfdii4omz7gcke5vq.b32.i2p:0 +ljsquuu3y4xje6l32p32inn6r2y6ull6oocgup6jtjrohrqxbz6a.b32.i2p:0 +lrah7acdsgopybg43shadwwiv6igezaw64i6jb5muqdg7dmhj3la.b32.i2p:0 +lzuu6mjtu7vd55d2biphicihufipoa7vyym6xfnkmmlra3tiziia.b32.i2p:0 +m6bpynxkv2ktwxkg6p2gyudjfhdupb6kuzabeqdnckkdkf4kxjla.b32.i2p:0 +m6v454xd6p3bt5swujgmveklsp7lzbkqlqqfc2p36cjlwv5dbucq.b32.i2p:0 +mlgeizrroynuhpxbzeosajt5u4ddcvynxfmcbm6kwjpaufilxigq.b32.i2p:0 +ofubxr2ir7u2guzjwyrvujicivzmvinwa36nuzlrg7tnsmebal7a.b32.i2p:0 +okfxeoh6itu4f5f43dhbzvkqwfrvm5c66lj6lvjj4q2b35i4pk4q.b32.i2p:0 +oz2ia3flpm3du2tyusulrn7h7e2eo3juzkrmn34bvnrlcrugv7ia.b32.i2p:0 +qd6jlsevsexww3wefpqs7iglxb3f63y4e6ydulfzrvwflpicmdqa.b32.i2p:0 +qddg7myylinn4tw6kdjmmp6fsyetkosnrbp2gsjx77tmkqyqv6ua.b32.i2p:0 +rizfinyses2r3or4iubs5wx66gdy6mpf73w7uobfacm2l5cral3q.b32.i2p:0 +s5hhjtmlg53bko3nwwskas7xgsmeqzy6thtsj5aa64djyrljgqaq.b32.i2p:0 +sedndhv5vpcgdmykyi5st4yqhdxl3hpdtglta4do435wupahhx6q.b32.i2p:0 +tsl4dlpu2id252b6crbdnblruct664se6f2iw35fuqwa3te7wcoq.b32.i2p:0 +tugq6wa2ls2bv27pr2iy3da3k5ow3fzefbcvjcr22uc7w5vmevja.b32.i2p:0 +usztavbib756k5vqggzgkyswoj6mttihjvp3c2pa642t2mb4pvsa.b32.i2p:0 +vgu6llqbyjphml25umd5ztvyxrxuplz2g74fzbx75g3kkaetoyiq.b32.i2p:0 +wjrul5jwwb4vqdmkkrjbmly7osj6amecdpsac5xvaoqrti4nb3ha.b32.i2p:0 +wvktcp7hy4l6immhi5cxyz2dlsbhhvtcmskjemrnqehacnoap23q.b32.i2p:0 +wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 +xlqndzjoe5nr2nsxo6xwibh44ghyz4jfqevu62xykvemextpmjbq.b32.i2p:0 +yc4xwin5ujenvcr6ynwkz7lnmmq3nmzxvfguele6ovqqpxgjvonq.b32.i2p:0 +zdoabsg7ugzothyawodjhq54nvlofa746rxfkxpnjzj6nukmha6a.b32.i2p:0 +zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 +zysrlpii5ftrzivfcyhdrwpeyyqddbrdefnfu5q6otk5gtugmh2a.b32.i2p:0 + +# manually added 2022-01 for minimal cjdns bootstrap support +[fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa]:8333 +[fcc7:be49:ccd1:dc91:3125:f0da:457d:8ce]:8333 diff --git a/contrib/seeds/nodes_test.txt b/contrib/seeds/nodes_test.txt index 118bec280e21..5b04791d6038 100644 --- a/contrib/seeds/nodes_test.txt +++ b/contrib/seeds/nodes_test.txt @@ -1,16 +1,89 @@ # List of fixed seed nodes for testnet -# Onion nodes -35k2va6vyw4oo5ly2quvcszgdqr56kcnfgcqpnpcffut4jn3mhhwgbid.onion:18333 -blo2esfvk2rr7sr4jspmu3vt2vpgr5rigflsj645fnku7v4qmljurtid.onion:18333 -fuckcswupr5rmlvx2kqqrrosxvjyong4hatmuvxsvtcwe4dsh5rus7qd.onion:18333 -gblylyacjlitd2ywdmo2qqylwtdky7kgeqfvlhiw4zdag4x62tx54hyd.onion:18333 -gzwpduv33l7yze3bcdzj3inebiyjwddjnwvnjhh5wvnv4me76mjt2kad.onion:18333 -h3rphzofxzq52tb63mg5f6kc4my3fkcrgh3m5qryeatts43iljbawiid.onion:18333 -kf4qlhek34b3kgyxyodlmvgm4bxfrjsbjtgayyaiuyhr2eoyfgtm3bad.onion:18333 +# Onion nodes, last verified 2022-08 for minimal torv3 bootstrap support +24j74ahq6ed4wmfrghdwroyfzimlkhnrb7zh4zw3vl2allzxbjrhaqid.onion:18333 +2fy74te65gm3c3gv3u5mhwdudvbdfh6k5fdz4gduimrltjjrxftbxrqd.onion:18333 +2lsncqdflwk272dhydrxf7ikfy23ppnmm54dnynyxiym6lqf3wowrmqd.onion:18333 +33o6qaidta7s2pmltet6vynd337vamgcifhh44rehwwxqpflcjt2njid.onion:18333 +3oo6bsc5mvf6a6ypmoaikilta6ka7mbdhdwhrnqhuhjlbaxyedvfvaqd.onion:18333 +3pe3fyklipy4sppkkgnhc22kcxtt57uler5kv72t676bbrwmcseo5qad.onion:18333 +4u4mcz2sfvxs7pwcwncswgmmcdzqtzjx7ztfo332jv4pqucb22ikdhad.onion:18333 +5v3i2kfqiqwp75gznjoptss7qgrcgseceqxpzpqkd34qeqzrg726i7id.onion:18333 +5zlrxk6q24t4vz5k4ie7gtuasdjavhoelhinzimxbfhc77u7vafipsid.onion:18333 +67s3af64ehw7xnxv422axm7tns4d6kutrftc6bjq375n74q3kj4pp7ad.onion:18333 +6a4ony53julvnufo632ktgmwvhupz63wbdwx7n7qudjy32qyq6gm3bqd.onion:18333 +6ftyg3nhc6tn2hyzls6zfdsfbroczhkxtdqumqb5q4yafhy5rdpapbid.onion:18333 +7554uw5djruh34j5ddx3iprzgqgzypcjtptwoldymfbgoywqcw2wiwyd.onion:18333 +766lozlabxaqjpbqsvt6sn3c65n6gkwwhoxyvggj7nfwnmw4cpaoccad.onion:18333 +7blv5abnytdf47yvbhxmykprmvjryqob65i2jmdwq3rrajcn2iiysbqd.onion:18333 +7v2ja4igx4v5y2jr6jrr6gaxohjhlzhvgwe4avlraxchozf7ea3kruqd.onion:18333 +7zgbmtzxow2oevd5aaqtsormw7ujv4zprl3oi2355immhq4gk7cyw5ad.onion:18333 +adstabjz7ec2y3jt4w2dvummowzv7g6m2f3kajeejffuaz7ojwj6epqd.onion:18333 +aesy6tfufadkut6flu2bsqgnw2422ur2ynjalguxlzuzuktg3zehttqd.onion:18333 +alxo32b5edi3bn2e224qrgytgxxpic4knyipvpdvctfsrvcaiq5lgeyd.onion:18333 +aoeart34umoonvd2kbqr3bc4sweu6a4msh2gp4skyqvei3shzcxbgmyd.onion:18333 +aprzvj7hgctsde4mkj3ewq35gvykspjvkqiygg7bpnw5tkvse2n7rhid.onion:18333 +awpk6z3xghx6ozouhodcydaqtr6uzzbnw4creuix7mkupxoxlmhhspad.onion:18333 +ayynqazucyh2jd5rehcfggmhunqpdwzlbhzbqgy6lj4ctz2ocj7chpid.onion:18333 +b2ika53aqckv4gs7wmog3byrea2vfzm5p7ye33digcsmvvnpbyqmzoyd.onion:18333 +be7zx3hh6dlahorlvsrrgqm4oahfrgqm2tbwnbd4u53ntu5f765n6hyd.onion:18333 +bluk62wj24bsvdwh47muo54hhwsatkftiqxevt5kba7hstjoex6ueeyd.onion:18333 +bubm6fiopfzkxqrfx6vqpioe5ahlhyubz57ogsqqy4ha5pnngiqlh6id.onion:18333 +d3czabzjj57lgrsr5gawkjd7v3gznrqa7zyizqmk4lryascavmipnyad.onion:18333 +ddj4cuvb32ve5chtp6jattcdnnmxmpoofjthzi7thgxxht7yqoetj3yd.onion:18333 +dqhhlssfwmh3g6zhwxpcfbw64xz5rfikcglinbhoxv5ajv4qzicjyeid.onion:18333 +drthcyb4x4rdfekw5g7xjogxi7aqoluilgulbgwvsme3nw3oibvchbad.onion:18333 +dwb47cmqa2tjpmvjaear7gdcars2lez6niefhi4qf22qehtyta6577qd.onion:18333 +e7tkrf54ng3q5vcn5gn77zwjwm74lkfav4mwdux3pvon6yvqg3tf46qd.onion:18333 +etuymy47s3quepvdaoo72i5e5mc7uovrzu5m4jf5q6mwlwizoxy4xgid.onion:18333 +fbimesnyhzubbzqc3uaufzkbyfmnkxvypoxaveaub7rzpzh2foxrn2yd.onion:18333 +fzbrwmgwmko7quelrhfuskt3ijabac76zx7g52dfrevmhdkj6ivh7qyd.onion:18333 +gy6nih4pmp5esyvvnhlj6qvk7zkbjuoswkxffyiip3dbkvsfxwz5zcqd.onion:18333 +ha62ziqzqdogd75zg7lfh4fqrg3bim3cpqzyupo43w5pw4fen6nr2pyd.onion:18333 +hacjjgj2mbqqrthzimmi6anvin7dljjhfl3ik6ebg3w3nmgsvr3ymmqd.onion:18333 +hbkp5xwpqo4qm75kpglfrclyiuuvdgv7mtiqfys7oqks4dmpqgpeoeid.onion:18333 +hqgoy62hoqjmz37brdfvoeov3cix5fixbqjoert4ydr6herg5oc3iwyd.onion:18333 +hvbmmzvqrpgps2x5u4ip4ksf3e5m2fneac754gtnhjn2rsevni6cz3ad.onion:18333 +hw3vzp32w4h6giplue6ix445oi6wt7gmeksrznb7tdfwhkgit7gnbbad.onion:18333 +iddr66ewkhenivapgianudjkwqcp6dxtssg7ixrdot5az6uh7m5tmjqd.onion:18333 +imya36iexiiiqrkwuxxcehnv4kg5shtirwd2vg4cnjy6lfjlph3fusqd.onion:18333 +iuhhuocns7entrzlxsxktyz2ibs7hqgiggv6sauzqkzka6laslwz7oqd.onion:18333 +ji5wmshokuc63eiulzlwj2zdvnligvrwfvvc76bice3tu43wfzvpmkyd.onion:18333 +jjfuyj7krgzkmpxvn3b2j2hwlzkmze3ezy3ifwk7dnswwawgmzqhjrqd.onion:18333 +jn2p4sgfphkxpow7kjrubrbqat77kkibzqkvuwhxyalcrazwmcqeaqyd.onion:18333 +jrveyz4us6sog6e6czsvr5mvvhgzjgv4idbe4idrolmqeudvt5a2dgid.onion:18333 +jsc4frvvnl2d3bhzyofsc72xpztgm23nl4fnb4dwkzsxr6fhij2q5iyd.onion:18333 +klymxdvje7kccv3tznabo3udopsftkmjemkbi2urqxjm4hefaudejjyd.onion:18333 +kwjxlauwjtecjfsiwopbl5pvn5n6z5rz76uk6osmlurd3uyuymcw7aid.onion:18333 +lc7upz2srw2yhpcvwg4afy64ylcoo6mfwlttqj5ovuglqnhnohpi5iqd.onion:18333 +lf3mpxfyjuovcqdvinl52pvdmmda6xqyfeiarlfamdjpgy3ouzmmlbyd.onion:18333 mc7k47ndjvvhcgs54wmjzxvate4rtuybbjoryikdssjhcxlx27psbyqd.onion:18333 -mrhiniicugfo7mgrwv3wtolk3tptlcw2uq7ih6sq43fa4k4zbilut3yd.onion:18333 -uiudyws3qizgmepfoh7wwjmsoxoxut4qrmotjjhrn247xnjopr7sfcid.onion:18333 -zc2wvoqcezcrf64trji6jmhtss34a5ds5ntzdhqegzvex3ynrd7nxcad.onion:18333 -zd5m3dgdn46naj36pxvvcalfw2paecle6sdxq64ptwxtxjomkywpklqd.onion:18333 - +mjbg3ggeuelmc7ixty3zjccyo2urg2uyherfqe7ytkm2ejkwlec7h6ad.onion:18333 +nkyqozv6kdwi423s7s2mezzguf5bafot2a3hv4ed2dbvtblisdmad4qd.onion:18333 +nvvqo4xxiwgb3y246jmcbuuveurfdq2zs3a5y7veqkeqv5jfhang7gyd.onion:18333 +o6vfovqxz3oxszfppczpjejwouobztjrgvfojc3emvhan3bkyskzhuad.onion:18333 +oaiw2lnhzgp5ry7ivzneuufmh7lfploquu2rjv5rozmlbefedsnxe5qd.onion:18333 +oln7ybci53wk4g5n42nipyixvyjxbludsbrfsmhnirb6tk7ovlikd5id.onion:18333 +otmfnhc6wrrbf2tpdy6zkisqc3r3urnsuowsnmatoto6yixaocnkseid.onion:18333 +ovc6sajbqfcbwv3wrq7ylklu6q6prvisz4jr4lyycn4kgukzjfe4mjad.onion:18333 +pm57didyzg5ljuvn5ufr5uun2iencuk3af2gzqc5zvgfh452c3rxtjyd.onion:18333 +pmismhpwug34gnqzbutranvx2wjwbshyqj4un2dyzyuvak2eh55psfyd.onion:18333 +polarisultijjhaku6z6u7jyboho5epdsg44ttebfaxmgau2z5sqolad.onion:18333 +qe2jbe447he6panfvpyqhyntf7346gmuf55bxrmdzggmgwyjsyknhxyd.onion:18333 +qz6yd5lsgdajcteoareeptwnipxsezyx5kks6ukpk5tvqinffzunqmyd.onion:18333 +rp6pn3b3oesyr2giolbysbjhqeugxntsu7crnkth4y33ok4zvcl7yrqd.onion:18333 +ujdchuw3hz5gkbouiv4p6pwbfdn7v4k6gluwvd4wiukqc7y7ow754uad.onion:18333 +vctlwaqgmu53eutz2hewuakcipfgtyljsd7czut4dd62xr3rp6fqezad.onion:18333 +vf5ur53tzmdtotvkndcgochklnuav7quqjvkc6mctqfvef6wnmn26mid.onion:18333 +wnxgjgjgplv5iu4mssyuunycvku4qnqr5t4q6cfdt47k7uwrfifuirad.onion:18333 +wpkbkdr7clw7zk3jkwiult6bf422j54u77ml4rgig2xq7icogyrcspid.onion:18333 +wzpdt24tdark26eugredddorik3tqwcj5ialtt2yim4ceiuiq7phkyqd.onion:18333 +xgapnikkbldoggjh5ewxkyauhuwnvf3xkspxroe3ojvfrk4lswkyx5yd.onion:18333 +xkvzdhcirontixbq6pjhru57bf4sgtqylvphk25csfrsy5p5ay3oc3yd.onion:18333 +xnipauenw5wnjb2zbx6v6umgvbb3g6xhf5kjo7pnyn5tdzvzaxtzicid.onion:18333 +yda7kwpii33j2qpq32ftf6lp22znknswipjwaccvsqj7l337jvfesnid.onion:18333 +z3j5foswuhpmtrg3kb56stkzmuoaesvd5jz3eztq46c4cidapglcyuad.onion:18333 +zcep44k7unwjm2wxty4ijh2e4fv5zgbrvwlctzyaqnrqhltjfzrtodad.onion:18333 +zmvizz7fd5hdue6wt3lwqumd6qwt4ijymmmotfzh75curq3mzjm53hyd.onion:18333 +zoaa3x7quyuijggii5zl4uyeioodudsgtr2uyv2qtdsslac5ukiwlxid.onion:18333 +zovauxlorl5eswumbsoxv2m5y3sm3qlk7657dcpr2uld7xf35en46sqd.onion:18333 diff --git a/contrib/seeds/suspicious_hosts.txt b/contrib/seeds/suspicious_hosts.txt deleted file mode 100644 index 13385cc81620..000000000000 --- a/contrib/seeds/suspicious_hosts.txt +++ /dev/null @@ -1,16 +0,0 @@ -130.211.129.106 -148.251.238.178 -176.9.46.6 -178.63.107.226 -54.173.72.127 -54.174.10.182 -54.183.64.54 -54.194.231.211 -54.66.214.167 -54.66.220.137 -54.67.33.14 -54.77.251.214 -54.94.195.96 -54.94.200.247 -83.81.130.26 -88.198.17.7 \ No newline at end of file diff --git a/contrib/signet/getcoins.py b/contrib/signet/getcoins.py index 691f0bb1b601..a069f5fad3de 100755 --- a/contrib/signet/getcoins.py +++ b/contrib/signet/getcoins.py @@ -1,36 +1,158 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse -import subprocess +import io import requests +import subprocess import sys +import xml.etree.ElementTree + +DEFAULT_GLOBAL_FAUCET = 'https://signetfaucet.com/claim' +DEFAULT_GLOBAL_CAPTCHA = 'https://signetfaucet.com/captcha' +GLOBAL_FIRST_BLOCK_HASH = '00000086d6b2636cb2a392d45edc4ec544a10024d30141c9adf4bfd9de533b53' + +# braille unicode block +BASE = 0x2800 +BIT_PER_PIXEL = [ + [0x01, 0x08], + [0x02, 0x10], + [0x04, 0x20], + [0x40, 0x80], +] +BW = 2 +BH = 4 + +# imagemagick or compatible fork (used for converting SVG) +CONVERT = 'convert' + +class PPMImage: + ''' + Load a PPM image (Pillow-ish API). + ''' + def __init__(self, f): + if f.readline() != b'P6\n': + raise ValueError('Invalid ppm format: header') + line = f.readline() + (width, height) = (int(x) for x in line.rstrip().split(b' ')) + if f.readline() != b'255\n': + raise ValueError('Invalid ppm format: color depth') + data = f.read(width * height * 3) + stride = width * 3 + self.size = (width, height) + self._grid = [[tuple(data[stride * y + 3 * x:stride * y + 3 * (x + 1)]) for x in range(width)] for y in range(height)] + + def getpixel(self, pos): + return self._grid[pos[1]][pos[0]] + +def print_image(img, threshold=128): + '''Print black-and-white image to terminal in braille unicode characters.''' + x_blocks = (img.size[0] + BW - 1) // BW + y_blocks = (img.size[1] + BH - 1) // BH + + for yb in range(y_blocks): + line = [] + for xb in range(x_blocks): + ch = BASE + for y in range(BH): + for x in range(BW): + try: + val = img.getpixel((xb * BW + x, yb * BH + y)) + except IndexError: + pass + else: + if val[0] < threshold: + ch |= BIT_PER_PIXEL[y][x] + line.append(chr(ch)) + print(''.join(line)) parser = argparse.ArgumentParser(description='Script to get coins from a faucet.', epilog='You may need to start with double-dash (--) when providing bitcoin-cli arguments.') parser.add_argument('-c', '--cmd', dest='cmd', default='bitcoin-cli', help='bitcoin-cli command to use') -parser.add_argument('-f', '--faucet', dest='faucet', default='https://signetfaucet.com/claim', help='URL of the faucet') +parser.add_argument('-f', '--faucet', dest='faucet', default=DEFAULT_GLOBAL_FAUCET, help='URL of the faucet') +parser.add_argument('-g', '--captcha', dest='captcha', default=DEFAULT_GLOBAL_CAPTCHA, help='URL of the faucet captcha, or empty if no captcha is needed') parser.add_argument('-a', '--addr', dest='addr', default='', help='Bitcoin address to which the faucet should send') parser.add_argument('-p', '--password', dest='password', default='', help='Faucet password, if any') +parser.add_argument('-n', '--amount', dest='amount', default='0.001', help='Amount to request (0.001-0.1, default is 0.001)') +parser.add_argument('-i', '--imagemagick', dest='imagemagick', default=CONVERT, help='Path to imagemagick convert utility') parser.add_argument('bitcoin_cli_args', nargs='*', help='Arguments to pass on to bitcoin-cli (default: -signet)') args = parser.parse_args() +if args.bitcoin_cli_args == []: + args.bitcoin_cli_args = ['-signet'] + + +def bitcoin_cli(rpc_command_and_params): + argv = [args.cmd] + args.bitcoin_cli_args + rpc_command_and_params + try: + return subprocess.check_output(argv).strip().decode() + except FileNotFoundError: + raise SystemExit(f"The binary {args.cmd} could not be found") + except subprocess.CalledProcessError: + cmdline = ' '.join(argv) + raise SystemExit(f"-----\nError while calling {cmdline} (see output above).") + + +if args.faucet.lower() == DEFAULT_GLOBAL_FAUCET: + # Get the hash of the block at height 1 of the currently active signet chain + curr_signet_hash = bitcoin_cli(['getblockhash', '1']) + if curr_signet_hash != GLOBAL_FIRST_BLOCK_HASH: + raise SystemExit('The global faucet cannot be used with a custom Signet network. Please use the global signet or setup your custom faucet to use this functionality.\n') +else: + # For custom faucets, don't request captcha by default. + if args.captcha == DEFAULT_GLOBAL_CAPTCHA: + args.captcha = '' + if args.addr == '': - if args.bitcoin_cli_args == []: - args.bitcoin_cli_args = ['-signet'] # get address for receiving coins + args.addr = bitcoin_cli(['getnewaddress', 'faucet', 'bech32']) + +data = {'address': args.addr, 'password': args.password, 'amount': args.amount} + +# Store cookies +# for debugging: print(session.cookies.get_dict()) +session = requests.Session() + +if args.captcha != '': # Retrieve a captcha try: - args.addr = subprocess.check_output([args.cmd] + args.bitcoin_cli_args + ['getnewaddress', 'faucet', 'bech32']).strip() + res = session.get(args.captcha) + res.raise_for_status() + except requests.exceptions.RequestException as e: + raise SystemExit(f"Unexpected error when contacting faucet: {e}") + + # Size limitation + svg = xml.etree.ElementTree.fromstring(res.content) + if svg.attrib.get('width') != '150' or svg.attrib.get('height') != '50': + raise SystemExit("Captcha size doesn't match expected dimensions 150x50") + + # Convert SVG image to PPM, and load it + try: + rv = subprocess.run([args.imagemagick, 'svg:-', '-depth', '8', 'ppm:-'], input=res.content, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except FileNotFoundError: - print('The binary', args.cmd, 'could not be found.') - exit() + raise SystemExit(f"The binary {args.imagemagick} could not be found. Please make sure ImageMagick (or a compatible fork) is installed and that the correct path is specified.") + + img = PPMImage(io.BytesIO(rv.stdout)) + + # Terminal interaction + print_image(img) + print(f"Captcha from URL {args.captcha}") + data['captcha'] = input('Enter captcha: ') -data = {'address': args.addr, 'password': args.password} try: - res = requests.post(args.faucet, data=data) + res = session.post(args.faucet, data=data) except: - print('Unexpected error when contacting faucet:', sys.exc_info()[0]) - exit() -print(res.text) + raise SystemExit(f"Unexpected error when contacting faucet: {sys.exc_info()[0]}") + +# Display the output as per the returned status code +if res: + # When the return code is in between 200 and 400 i.e. successful + print(res.text) +elif res.status_code == 404: + print('The specified faucet URL does not exist. Please check for any server issues/typo.') +elif res.status_code == 429: + print('The script does not allow for repeated transactions as the global faucet is rate-limitied to 1 request/IP/day. You can access the faucet website to get more coins manually') +else: + print(f'Returned Error Code {res.status_code}\n{res.text}\n') + print('Please check the provided arguments for their validity and/or any possible typo.') diff --git a/contrib/signet/miner b/contrib/signet/miner index 78e1fa5ecd26..61d9f62be718 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -4,26 +4,23 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse -import base64 import json import logging import math -import os.path +import os import re import struct import sys import time import subprocess -from binascii import unhexlify -from io import BytesIO - PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional")) sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL) -from test_framework.blocktools import WITNESS_COMMITMENT_HEADER, script_BIP34_coinbase_height # noqa: E402 -from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, hash256, ser_compact_size, ser_string, ser_uint256, tx_from_hex, uint256_from_str # noqa: E402 +from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height # noqa: E402 +from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_binary, from_hex, ser_string, ser_uint256, tx_from_hex # noqa: E402 +from test_framework.psbt import PSBT, PSBTMap, PSBT_GLOBAL_UNSIGNED_TX, PSBT_IN_FINAL_SCRIPTSIG, PSBT_IN_FINAL_SCRIPTWITNESS, PSBT_IN_NON_WITNESS_UTXO, PSBT_IN_SIGHASH_TYPE # noqa: E402 from test_framework.script import CScriptOp # noqa: E402 logging.basicConfig( @@ -35,99 +32,12 @@ SIGNET_HEADER = b"\xec\xc7\xda\xa2" PSBT_SIGNET_BLOCK = b"\xfc\x06signetb" # proprietary PSBT global field holding the block being signed RE_MULTIMINER = re.compile("^(\d+)(-(\d+))?/(\d+)$") -# #### some helpers that could go into test_framework - -# like from_hex, but without the hex part -def FromBinary(cls, stream): - """deserialize a binary stream (or bytes object) into an object""" - # handle bytes object by turning it into a stream - was_bytes = isinstance(stream, bytes) - if was_bytes: - stream = BytesIO(stream) - obj = cls() - obj.deserialize(stream) - if was_bytes: - assert len(stream.read()) == 0 - return obj - -class PSBTMap: - """Class for serializing and deserializing PSBT maps""" - - def __init__(self, map=None): - self.map = map if map is not None else {} - - def deserialize(self, f): - m = {} - while True: - k = deser_string(f) - if len(k) == 0: - break - v = deser_string(f) - if len(k) == 1: - k = k[0] - assert k not in m - m[k] = v - self.map = m - - def serialize(self): - m = b"" - for k,v in self.map.items(): - if isinstance(k, int) and 0 <= k and k <= 255: - k = bytes([k]) - m += ser_compact_size(len(k)) + k - m += ser_compact_size(len(v)) + v - m += b"\x00" - return m - -class PSBT: - """Class for serializing and deserializing PSBTs""" - - def __init__(self): - self.g = PSBTMap() - self.i = [] - self.o = [] - self.tx = None - - def deserialize(self, f): - assert f.read(5) == b"psbt\xff" - self.g = FromBinary(PSBTMap, f) - assert 0 in self.g.map - self.tx = FromBinary(CTransaction, self.g.map[0]) - self.i = [FromBinary(PSBTMap, f) for _ in self.tx.vin] - self.o = [FromBinary(PSBTMap, f) for _ in self.tx.vout] - return self - - def serialize(self): - assert isinstance(self.g, PSBTMap) - assert isinstance(self.i, list) and all(isinstance(x, PSBTMap) for x in self.i) - assert isinstance(self.o, list) and all(isinstance(x, PSBTMap) for x in self.o) - assert 0 in self.g.map - tx = FromBinary(CTransaction, self.g.map[0]) - assert len(tx.vin) == len(self.i) - assert len(tx.vout) == len(self.o) - - psbt = [x.serialize() for x in [self.g] + self.i + self.o] - return b"psbt\xff" + b"".join(psbt) - - def to_base64(self): - return base64.b64encode(self.serialize()).decode("utf8") - - @classmethod - def from_base64(cls, b64psbt): - return FromBinary(cls, base64.b64decode(b64psbt)) - -# ##### - def create_coinbase(height, value, spk): cb = CTransaction() cb.vin = [CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)] cb.vout = [CTxOut(value, spk)] return cb -def get_witness_script(witness_root, witness_nonce): - commitment = uint256_from_str(hash256(ser_uint256(witness_root) + ser_uint256(witness_nonce))) - return b"\x6a" + CScriptOp.encode_op_pushdata(WITNESS_COMMITMENT_HEADER + ser_uint256(commitment)) - def signet_txs(block, challenge): # assumes signet solution has not been added yet so does not need # to be removed @@ -164,11 +74,11 @@ def signet_txs(block, challenge): def do_createpsbt(block, signme, spendme): psbt = PSBT() - psbt.g = PSBTMap( {0: signme.serialize(), + psbt.g = PSBTMap( {PSBT_GLOBAL_UNSIGNED_TX: signme.serialize(), PSBT_SIGNET_BLOCK: block.serialize() } ) - psbt.i = [ PSBTMap( {0: spendme.serialize(), - 3: bytes([1,0,0,0])}) + psbt.i = [ PSBTMap( {PSBT_IN_NON_WITNESS_UTXO: spendme.serialize(), + PSBT_IN_SIGHASH_TYPE: bytes([1,0,0,0])}) ] psbt.o = [ PSBTMap() ] return psbt.to_base64() @@ -180,10 +90,10 @@ def do_decode_psbt(b64psbt): assert len(psbt.tx.vout) == 1 assert PSBT_SIGNET_BLOCK in psbt.g.map - scriptSig = psbt.i[0].map.get(7, b"") - scriptWitness = psbt.i[0].map.get(8, b"\x00") + scriptSig = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTSIG, b"") + scriptWitness = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTWITNESS, b"\x00") - return FromBinary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness + return from_binary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness def finish_block(block, signet_solution, grind_cmd): block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER + signet_solution) @@ -202,7 +112,7 @@ def finish_block(block, signet_solution, grind_cmd): def generate_psbt(tmpl, reward_spk, *, blocktime=None): signet_spk = tmpl["signet_challenge"] - signet_spk_bin = unhexlify(signet_spk) + signet_spk_bin = bytes.fromhex(signet_spk) cbtx = create_coinbase(height=tmpl["height"], value=tmpl["coinbasevalue"], spk=reward_spk) cbtx.vin[0].nSequence = 2**32-2 @@ -223,7 +133,7 @@ def generate_psbt(tmpl, reward_spk, *, blocktime=None): cbwit = CTxInWitness() cbwit.scriptWitness.stack = [ser_uint256(witnonce)] block.vtx[0].wit.vtxinwit = [cbwit] - block.vtx[0].vout.append(CTxOut(0, get_witness_script(witroot, witnonce))) + block.vtx[0].vout.append(CTxOut(0, bytes(get_witness_script(witroot, witnonce)))) signme, spendme = signet_txs(block, signet_spk_bin) @@ -258,7 +168,7 @@ def get_reward_addr_spk(args, height): return args.address, args.reward_spk reward_addr = get_reward_address(args, height) - reward_spk = unhexlify(json.loads(args.bcli("getaddressinfo", reward_addr))["scriptPubKey"]) + reward_spk = bytes.fromhex(json.loads(args.bcli("getaddressinfo", reward_addr))["scriptPubKey"]) if args.address is not None: # will always be the same, so cache args.reward_spk = reward_spk @@ -315,7 +225,7 @@ def seconds_to_hms(s): out = "-" + out return out -def next_block_delta(last_nbits, last_hash, ultimate_target, do_poisson): +def next_block_delta(last_nbits, last_hash, ultimate_target, do_poisson, max_interval): # strategy: # 1) work out how far off our desired target we are # 2) cap it to a factor of 4 since that's the best we can do in a single retarget period @@ -338,7 +248,7 @@ def next_block_delta(last_nbits, last_hash, ultimate_target, do_poisson): this_interval_variance = 1 this_interval = avg_interval * this_interval_variance - this_interval = max(1, min(this_interval, 3600)) + this_interval = max(1, min(this_interval, max_interval)) return this_interval @@ -398,6 +308,10 @@ def do_generate(args): return 1 my_blocks = (start-1, stop, total) + if args.max_interval < 960: + logging.error("--max-interval must be at least 960 (16 minutes)") + return 1 + ultimate_target = nbits_to_target(int(args.nbits,16)) mined_blocks = 0 @@ -414,7 +328,7 @@ def do_generate(args): if lastheader is None: lastheader = bestheader["hash"] elif bestheader["hash"] != lastheader: - next_delta = next_block_delta(int(bestheader["bits"], 16), bestheader["hash"], ultimate_target, args.poisson) + next_delta = next_block_delta(int(bestheader["bits"], 16), bestheader["hash"], ultimate_target, args.poisson, args.max_interval) next_delta += bestheader["time"] - time.time() next_is_mine = next_block_is_mine(bestheader["hash"], my_blocks) logging.info("Received new block at height %d; next in %s (%s)", bestheader["height"], seconds_to_hms(next_delta), ("mine" if next_is_mine else "backup")) @@ -428,14 +342,14 @@ def do_generate(args): action_time = now is_mine = True elif bestheader["height"] == 0: - time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson) + time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson, args.max_interval) time_delta *= 100 # 100 blocks logging.info("Backdating time for first block to %d minutes ago" % (time_delta/60)) mine_time = now - time_delta action_time = now is_mine = True else: - time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson) + time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson, args.max_interval) mine_time = bestheader["time"] + time_delta is_mine = next_block_is_mine(bci["bestblockhash"], my_blocks) @@ -494,10 +408,11 @@ def do_generate(args): logging.debug("Mining block delta=%s start=%s mine=%s", seconds_to_hms(mine_time-bestheader["time"]), mine_time, is_mine) mined_blocks += 1 psbt = generate_psbt(tmpl, reward_spk, blocktime=mine_time) - psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=psbt.encode('utf8'))) + input_stream = os.linesep.join([psbt, "true", "ALL"]).encode('utf8') + psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=input_stream)) if not psbt_signed.get("complete",False): logging.debug("Generated PSBT: %s" % (psbt,)) - sys.stderr.write("PSBT signing failed") + sys.stderr.write("PSBT signing failed\n") return 1 block, signet_solution = do_decode_psbt(psbt_signed["psbt"]) block = finish_block(block, signet_solution, args.grind_cmd) @@ -508,7 +423,7 @@ def do_generate(args): # report bstr = "block" if is_mine else "backup block" - next_delta = next_block_delta(block.nBits, block.hash, ultimate_target, args.poisson) + next_delta = next_block_delta(block.nBits, block.hash, ultimate_target, args.poisson, args.max_interval) next_delta += block.nTime - time.time() next_is_mine = next_block_is_mine(block.hash, my_blocks) @@ -586,6 +501,7 @@ def main(): generate.add_argument("--multiminer", default=None, type=str, help="Specify which set of blocks to mine (eg: 1-40/100 for the first 40%%, 2/3 for the second 3rd)") generate.add_argument("--backup-delay", default=300, type=int, help="Seconds to delay before mining blocks reserved for other miners (default=300)") generate.add_argument("--standby-delay", default=0, type=int, help="Seconds to delay before mining blocks (default=0)") + generate.add_argument("--max-interval", default=1800, type=int, help="Maximum interblock interval (seconds)") calibrate = cmds.add_parser("calibrate", help="Calibrate difficulty") calibrate.set_defaults(fn=do_calibrate) @@ -627,5 +543,3 @@ def main(): if __name__ == "__main__": main() - - diff --git a/contrib/testgen/README.md b/contrib/testgen/README.md index fcc5a378e2cc..2f0288df165b 100644 --- a/contrib/testgen/README.md +++ b/contrib/testgen/README.md @@ -2,7 +2,7 @@ Utilities to generate test vectors for the data-driven Bitcoin tests. -Usage: +To use inside a scripted-diff (or just execute directly): - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json + ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json + ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json diff --git a/contrib/testgen/base58.py b/contrib/testgen/base58.py deleted file mode 100644 index 87341ccf96dc..000000000000 --- a/contrib/testgen/base58.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2012-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Bitcoin base58 encoding and decoding. - -Based on https://bitcointalk.org/index.php?topic=1026.0 (public domain) -''' -import hashlib - -# for compatibility with following code... -class SHA256: - new = hashlib.sha256 - -if str != bytes: - # Python 3.x - def ord(c): - return c - def chr(n): - return bytes( (n,) ) - -__b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' -__b58base = len(__b58chars) -b58chars = __b58chars - -def b58encode(v): - """ encode v, which is a string of bytes, to base58. - """ - long_value = 0 - for (i, c) in enumerate(v[::-1]): - if isinstance(c, str): - c = ord(c) - long_value += (256**i) * c - - result = '' - while long_value >= __b58base: - div, mod = divmod(long_value, __b58base) - result = __b58chars[mod] + result - long_value = div - result = __b58chars[long_value] + result - - # Bitcoin does a little leading-zero-compression: - # leading 0-bytes in the input become leading-1s - nPad = 0 - for c in v: - if c == 0: - nPad += 1 - else: - break - - return (__b58chars[0]*nPad) + result - -def b58decode(v, length = None): - """ decode v into a string of len bytes - """ - long_value = 0 - for i, c in enumerate(v[::-1]): - pos = __b58chars.find(c) - assert pos != -1 - long_value += pos * (__b58base**i) - - result = bytes() - while long_value >= 256: - div, mod = divmod(long_value, 256) - result = chr(mod) + result - long_value = div - result = chr(long_value) + result - - nPad = 0 - for c in v: - if c == __b58chars[0]: - nPad += 1 - continue - break - - result = bytes(nPad) + result - if length is not None and len(result) != length: - return None - - return result - -def checksum(v): - """Return 32-bit checksum based on SHA256""" - return SHA256.new(SHA256.new(v).digest()).digest()[0:4] - -def b58encode_chk(v): - """b58encode a string, with 32-bit checksum""" - return b58encode(v + checksum(v)) - -def b58decode_chk(v): - """decode a base58 string, check and remove checksum""" - result = b58decode(v) - if result is None: - return None - if result[-4:] == checksum(result[:-4]): - return result[:-4] - else: - return None - -def get_bcaddress_version(strAddress): - """ Returns None if strAddress is invalid. Otherwise returns integer version of address. """ - addr = b58decode_chk(strAddress) - if addr is None or len(addr)!=21: - return None - version = addr[0] - return ord(version) - -if __name__ == '__main__': - # Test case (from http://gitorious.org/bitcoin/python-base58.git) - assert get_bcaddress_version('15VjRaDX9zpbA8LVnbrCAFzrVzN7ixHNsC') == 0 - _ohai = 'o hai'.encode('ascii') - _tmp = b58encode(_ohai) - assert _tmp == 'DYB3oMS' - assert b58decode(_tmp, 5) == _ohai - print("Tests passed") diff --git a/contrib/testgen/gen_key_io_test_vectors.py b/contrib/testgen/gen_key_io_test_vectors.py index 74918cfb048a..7bfb1d76a8b7 100755 --- a/contrib/testgen/gen_key_io_test_vectors.py +++ b/contrib/testgen/gen_key_io_test_vectors.py @@ -1,21 +1,21 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2020 The Bitcoin Core developers +# Copyright (c) 2012-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Generate valid and invalid base58/bech32(m) address and private key test vectors. - -Usage: - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json ''' -# 2012 Wladimir J. van der Laan -# Released under MIT License -import os + from itertools import islice -from base58 import b58encode_chk, b58decode_chk, b58chars +import os import random -from segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.address import base58_to_byte, byte_to_base58, b58chars # noqa: E402 +from test_framework.script import OP_0, OP_1, OP_2, OP_3, OP_16, OP_DUP, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_CHECKSIG # noqa: E402 +from test_framework.segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding # noqa: E402 # key types PUBKEY_ADDRESS = 0 @@ -29,16 +29,6 @@ PRIVKEY_REGTEST = 239 # script -OP_0 = 0x00 -OP_1 = 0x51 -OP_2 = 0x52 -OP_3 = 0x53 -OP_16 = 0x60 -OP_DUP = 0x76 -OP_EQUAL = 0x87 -OP_EQUALVERIFY = 0x88 -OP_HASH160 = 0xa9 -OP_CHECKSIG = 0xac pubkey_prefix = (OP_DUP, OP_HASH160, 20) pubkey_suffix = (OP_EQUALVERIFY, OP_CHECKSIG) script_prefix = (OP_HASH160, 20) @@ -114,8 +104,10 @@ def is_valid(v): '''Check vector v for validity''' if len(set(v) - set(b58chars)) > 0: return is_valid_bech32(v) - result = b58decode_chk(v) - if result is None: + try: + payload, version = base58_to_byte(v) + result = bytes([version]) + payload + except ValueError: # thrown if checksum doesn't match return is_valid_bech32(v) for template in templates: prefix = bytearray(template[0]) @@ -135,18 +127,19 @@ def is_valid_bech32(v): def gen_valid_base58_vector(template): '''Generate valid base58 vector''' prefix = bytearray(template[0]) - payload = bytearray(os.urandom(template[1])) + payload = rand_bytes(size=template[1]) suffix = bytearray(template[2]) dst_prefix = bytearray(template[4]) dst_suffix = bytearray(template[5]) - rv = b58encode_chk(prefix + payload + suffix) + assert len(prefix) == 1 + rv = byte_to_base58(payload + suffix, prefix[0]) return rv, dst_prefix + payload + dst_suffix def gen_valid_bech32_vector(template): '''Generate valid bech32 vector''' hrp = template[0] witver = template[1] - witprog = bytearray(os.urandom(template[2])) + witprog = rand_bytes(size=template[2]) encoding = template[4] dst_prefix = bytearray(template[5]) rv = bech32_encode(encoding, hrp, [witver] + convertbits(witprog, 8, 5)) @@ -176,21 +169,22 @@ def gen_invalid_base58_vector(template): corrupt_suffix = randbool(0.2) if corrupt_prefix: - prefix = os.urandom(1) + prefix = rand_bytes(size=1) else: prefix = bytearray(template[0]) if randomize_payload_size: - payload = os.urandom(max(int(random.expovariate(0.5)), 50)) + payload = rand_bytes(size=max(int(random.expovariate(0.5)), 50)) else: - payload = os.urandom(template[1]) + payload = rand_bytes(size=template[1]) if corrupt_suffix: - suffix = os.urandom(len(template[2])) + suffix = rand_bytes(size=len(template[2])) else: suffix = bytearray(template[2]) - val = b58encode_chk(prefix + payload + suffix) + assert len(prefix) == 1 + val = byte_to_base58(payload + suffix, prefix[0]) if random.randint(0,10)<1: # line corruption if randbool(): # add random character to end val += random.choice(b58chars) @@ -206,7 +200,7 @@ def gen_invalid_bech32_vector(template): to_upper = randbool(0.1) hrp = template[0] witver = template[1] - witprog = bytearray(os.urandom(template[2])) + witprog = rand_bytes(size=template[2]) encoding = template[3] if no_data: @@ -236,6 +230,9 @@ def randbool(p = 0.5): '''Return True with P(p)''' return random.random() < p +def rand_bytes(*, size): + return bytearray(random.getrandbits(8) for _ in range(size)) + def gen_invalid_vectors(): '''Generate invalid test vectors''' # start with some manual edge-cases @@ -250,9 +247,9 @@ def gen_invalid_vectors(): yield val, if __name__ == '__main__': - import sys import json iters = {'valid':gen_valid_vectors, 'invalid':gen_invalid_vectors} + random.seed(42) try: uiter = iters[sys.argv[1]] except IndexError: diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md new file mode 100644 index 000000000000..a409a23ef8db --- /dev/null +++ b/contrib/tracing/README.md @@ -0,0 +1,288 @@ +Example scripts for User-space, Statically Defined Tracing (USDT) +================================================================= + +This directory contains scripts showcasing User-space, Statically Defined +Tracing (USDT) support for Bitcoin Core on Linux using. For more information on +USDT support in Bitcoin Core see the [USDT documentation]. + +[USDT documentation]: ../../doc/tracing.md + + +Examples for the two main eBPF front-ends, [bpftrace] and +[BPF Compiler Collection (BCC)], with support for USDT, are listed. BCC is used +for complex tools and daemons and `bpftrace` is preferred for one-liners and +shorter scripts. + +[bpftrace]: https://github.com/iovisor/bpftrace +[BPF Compiler Collection (BCC)]: https://github.com/iovisor/bcc + + +To develop and run bpftrace and BCC scripts you need to install the +corresponding packages. See [installing bpftrace] and [installing BCC] for more +information. For development there exist a [bpftrace Reference Guide], a +[BCC Reference Guide], and a [bcc Python Developer Tutorial]. + +[installing bpftrace]: https://github.com/iovisor/bpftrace/blob/master/INSTALL.md +[installing BCC]: https://github.com/iovisor/bcc/blob/master/INSTALL.md +[bpftrace Reference Guide]: https://github.com/iovisor/bpftrace/blob/master/docs/reference_guide.md +[BCC Reference Guide]: https://github.com/iovisor/bcc/blob/master/docs/reference_guide.md +[bcc Python Developer Tutorial]: https://github.com/iovisor/bcc/blob/master/docs/tutorial_bcc_python_developer.md + +## Examples + +The bpftrace examples contain a relative path to the `bitcoind` binary. By +default, the scripts should be run from the repository-root and assume a +self-compiled `bitcoind` binary. The paths in the examples can be changed, for +example, to point to release builds if needed. See the +[Bitcoin Core USDT documentation] on how to list available tracepoints in your +`bitcoind` binary. + +[Bitcoin Core USDT documentation]: ../../doc/tracing.md#listing-available-tracepoints + +**WARNING: eBPF programs require root privileges to be loaded into a Linux +kernel VM. This means the bpftrace and BCC examples must be executed with root +privileges. Make sure to carefully review any scripts that you run with root +privileges first!** + +### log_p2p_traffic.bt + +A bpftrace script logging information about inbound and outbound P2P network +messages. Based on the `net:inbound_message` and `net:outbound_message` +tracepoints. + +By default, `bpftrace` limits strings to 64 bytes due to the limited stack size +in the eBPF VM. For example, Tor v3 addresses exceed the string size limit which +results in the port being cut off during logging. The string size limit can be +increased with the `BPFTRACE_STRLEN` environment variable (`BPFTRACE_STRLEN=70` +works fine). + +``` +$ bpftrace contrib/tracing/log_p2p_traffic.bt +``` + +Output +``` +outbound 'ping' msg to peer 11 (outbound-full-relay, [2a02:b10c:f747:1:ef:fake:ipv6:addr]:8333) with 8 bytes +inbound 'pong' msg from peer 11 (outbound-full-relay, [2a02:b10c:f747:1:ef:fake:ipv6:addr]:8333) with 8 bytes +inbound 'inv' msg from peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 37 bytes +outbound 'getdata' msg to peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 37 bytes +inbound 'tx' msg from peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 222 bytes +outbound 'inv' msg to peer 9 (outbound-full-relay, faketorv3addressa2ufa6odvoi3s77j4uegey0xb10csyfyve2t33curbyd.onion:8333) with 37 bytes +outbound 'inv' msg to peer 7 (outbound-full-relay, XX.XX.XXX.242:8333) with 37 bytes +… +``` + +### p2p_monitor.py + +A BCC Python script using curses for an interactive P2P message monitor. Based +on the `net:inbound_message` and `net:outbound_message` tracepoints. + +Inbound and outbound traffic is listed for each peer together with information +about the connection. Peers can be selected individually to view recent P2P +messages. + +``` +$ python3 contrib/tracing/p2p_monitor.py ./src/bitcoind +``` + +Lists selectable peers and traffic and connection information. +``` + P2P Message Monitor + Navigate with UP/DOWN or J/K and select a peer with ENTER or SPACE to see individual P2P messages + + PEER OUTBOUND INBOUND TYPE ADDR + 0 46 398 byte 61 1407590 byte block-relay-only XX.XX.XXX.196:8333 + 11 1156 253570 byte 3431 2394924 byte outbound-full-relay XXX.X.XX.179:8333 + 13 3425 1809620 byte 1236 305458 byte inbound XXX.X.X.X:60380 + 16 1046 241633 byte 1589 1199220 byte outbound-full-relay 4faketorv2pbfu7x.onion:8333 + 19 577 181679 byte 390 148951 byte outbound-full-relay kfake4vctorjv2o2.onion:8333 + 20 11 1248 byte 13 1283 byte block-relay-only [2600:fake:64d9:b10c:4436:aaaa:fe:bb]:8333 + 21 11 1248 byte 13 1299 byte block-relay-only XX.XXX.X.155:8333 + 22 5 103 byte 1 102 byte feeler XX.XX.XXX.173:8333 + 23 11 1248 byte 12 1255 byte block-relay-only XX.XXX.XXX.220:8333 + 24 3 103 byte 1 102 byte feeler XXX.XXX.XXX.64:8333 +… +``` + +Showing recent P2P messages between our node and a selected peer. + +``` + ---------------------------------------------------------------------- + | PEER 16 (4faketorv2pbfu7x.onion:8333) | + | OUR NODE outbound-full-relay PEER | + | <--- sendcmpct (9 bytes) | + | inv (37 byte) ---> | + | <--- ping (8 bytes) | + | pong (8 byte) ---> | + | inv (37 byte) ---> | + | <--- addr (31 bytes) | + | inv (37 byte) ---> | + | <--- getheaders (1029 bytes) | + | headers (1 byte) ---> | + | <--- feefilter (8 bytes) | + | <--- pong (8 bytes) | + | <--- headers (82 bytes) | + | <--- addr (30003 bytes) | + | inv (1261 byte) ---> | + | … | + +``` + +### log_raw_p2p_msgs.py + +A BCC Python script showcasing eBPF and USDT limitations when passing data +larger than about 32kb. Based on the `net:inbound_message` and +`net:outbound_message` tracepoints. + +Bitcoin P2P messages can be larger than 32kb (e.g. `tx`, `block`, ...). The +eBPF VM's stack is limited to 512 bytes, and we can't allocate more than about +32kb for a P2P message in the eBPF VM. The **message data is cut off** when the +message is larger than MAX_MSG_DATA_LENGTH (see script). This can be detected +in user-space by comparing the data length to the message length variable. The +message is cut off when the data length is smaller than the message length. +A warning is included with the printed message data. + +Data is submitted to user-space (i.e. to this script) via a ring buffer. The +throughput of the ring buffer is limited. Each p2p_message is about 32kb in +size. In- or outbound messages submitted to the ring buffer in rapid +succession fill the ring buffer faster than it can be read. Some messages are +lost. BCC prints: `Possibly lost 2 samples` on lost messages. + + +``` +$ python3 contrib/tracing/log_raw_p2p_msgs.py ./src/bitcoind +``` + +``` +Logging raw P2P messages. +Messages larger that about 32kb will be cut off! +Some messages might be lost! + outbound msg 'inv' from peer 4 (outbound-full-relay, XX.XXX.XX.4:8333) with 253 bytes: 0705000000be2245c8f844c9f763748e1a7… +… +Warning: incomplete message (only 32568 out of 53552 bytes)! inbound msg 'tx' from peer 32 (outbound-full-relay, XX.XXX.XXX.43:8333) with 53552 bytes: 020000000001fd3c01939c85ad6756ed9fc… +… +Possibly lost 2 samples +``` + +### connectblock_benchmark.bt + +A `bpftrace` script to benchmark the `ConnectBlock()` function during, for +example, a blockchain re-index. Based on the `validation:block_connected` USDT +tracepoint. + +The script takes three positional arguments. The first two arguments, the start, +and end height indicate between which blocks the benchmark should be run. The +third acts as a duration threshold in milliseconds. When the `ConnectBlock()` +function takes longer than the threshold, information about the block, is +printed. For more details, see the header comment in the script. + +The following command can be used to benchmark, for example, `ConnectBlock()` +between height 20000 and 38000 on SigNet while logging all blocks that take +longer than 25ms to connect. + +``` +$ bpftrace contrib/tracing/connectblock_benchmark.bt 20000 38000 25 +``` + +In a different terminal, starting Bitcoin Core in SigNet mode and with +re-indexing enabled. + +``` +$ ./src/bitcoind -signet -reindex +``` + +This produces the following output. +``` +Attaching 5 probes... +ConnectBlock Benchmark between height 20000 and 38000 inclusive +Logging blocks taking longer than 25 ms to connect. +Starting Connect Block Benchmark between height 20000 and 38000. +BENCH 39 blk/s 59 tx/s 59 inputs/s 20 sigops/s (height 20038) +Block 20492 (000000f555653bb05e2f3c6e79925e01a20dd57033f4dc7c354b46e34735d32b) 20 tx 2319 ins 2318 sigops took 38 ms +BENCH 1840 blk/s 2117 tx/s 4478 inputs/s 2471 sigops/s (height 21879) +BENCH 1816 blk/s 4972 tx/s 4982 inputs/s 125 sigops/s (height 23695) +BENCH 2095 blk/s 2890 tx/s 2910 inputs/s 152 sigops/s (height 25790) +BENCH 1684 blk/s 3979 tx/s 4053 inputs/s 288 sigops/s (height 27474) +BENCH 1155 blk/s 3216 tx/s 3252 inputs/s 115 sigops/s (height 28629) +BENCH 1797 blk/s 2488 tx/s 2503 inputs/s 111 sigops/s (height 30426) +BENCH 1849 blk/s 6318 tx/s 6569 inputs/s 12189 sigops/s (height 32275) +BENCH 946 blk/s 20209 tx/s 20775 inputs/s 83809 sigops/s (height 33221) +Block 33406 (0000002adfe4a15cfcd53bd890a89bbae836e5bb7f38bac566f61ad4548c87f6) 25 tx 2045 ins 2090 sigops took 29 ms +Block 33687 (00000073231307a9828e5607ceb8156b402efe56747271a4442e75eb5b77cd36) 52 tx 1797 ins 1826 sigops took 26 ms +BENCH 582 blk/s 21581 tx/s 27673 inputs/s 60345 sigops/s (height 33803) +BENCH 1035 blk/s 19735 tx/s 19776 inputs/s 51355 sigops/s (height 34838) +Block 35625 (0000006b00b347390c4768ea9df2655e9ff4b120f29d78594a2a702f8a02c997) 20 tx 3374 ins 3371 sigops took 49 ms +BENCH 887 blk/s 17857 tx/s 22191 inputs/s 24404 sigops/s (height 35725) +Block 35937 (000000d816d13d6e39b471cd4368db60463a764ba1f29168606b04a22b81ea57) 75 tx 3943 ins 3940 sigops took 61 ms +BENCH 823 blk/s 16298 tx/s 21031 inputs/s 18440 sigops/s (height 36548) +Block 36583 (000000c3e260556dbf42968aae3f904dba8b8c1ff96a6f6e3aa5365d2e3ad317) 24 tx 2198 ins 2194 sigops took 34 ms +Block 36700 (000000b3b173de9e65a3cfa738d976af6347aaf83fa17ab3f2a4d2ede3ddfac4) 73 tx 1615 ins 1611 sigops took 31 ms +Block 36832 (0000007859578c02c1ac37dabd1b9ec19b98f350b56935f5dd3a41e9f79f836e) 34 tx 1440 ins 1436 sigops took 26 ms +BENCH 613 blk/s 16718 tx/s 25074 inputs/s 23022 sigops/s (height 37161) +Block 37870 (000000f5c1086291ba2d943fb0c3bc82e71c5ee341ee117681d1456fbf6c6c38) 25 tx 1517 ins 1514 sigops took 29 ms +BENCH 811 blk/s 16031 tx/s 20921 inputs/s 18696 sigops/s (height 37972) + +Took 14055 ms to connect the blocks between height 20000 and 38000. + +Histogram of block connection times in milliseconds (ms). +@durations: +[0] 16838 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@| +[1] 882 |@@ | +[2, 4) 236 | | +[4, 8) 23 | | +[8, 16) 9 | | +[16, 32) 9 | | +[32, 64) 4 | | +``` + +### log_utxocache_flush.py + +A BCC Python script to log the UTXO cache flushes. Based on the +`utxocache:flush` tracepoint. + +```bash +$ python3 contrib/tracing/log_utxocache_flush.py ./src/bitcoind +``` + +``` +Logging utxocache flushes. Ctrl-C to end... +Duration (µs) Mode Coins Count Memory Usage Prune +730451 IF_NEEDED 22990 3323.54 kB True +637657 ALWAYS 122320 17124.80 kB False +81349 ALWAYS 0 1383.49 kB False +``` + +### log_utxos.bt + +A `bpftrace` script to log information about the coins that are added, spent, or +uncached from the UTXO set. Based on the `utxocache:add`, `utxocache:spend` and +`utxocache:uncache` tracepoints. + +```bash +$ bpftrace contrib/tracing/log_utxos.bt +``` + +This should produce an output similar to the following. If you see bpftrace +warnings like `Lost 24 events`, the eBPF perf ring-buffer is filled faster +than it is being read. You can increase the ring-buffer size by setting the +ENV variable `BPFTRACE_PERF_RB_PAGES` (default 64) at a cost of higher +memory usage. See the [bpftrace reference guide] for more information. + +[bpftrace reference guide]: https://github.com/iovisor/bpftrace/blob/master/docs/reference_guide.md#98-bpftrace_perf_rb_pages + +```bash +Attaching 4 probes... +OP Outpoint Value Height Coinbase +Added 6ba9ad857e1ef2eb2a2c94f06813c414c7ab273e3d6bd7ad64e000315a887e7c:1 10000 2094512 No +Spent fa7dc4db56637a151f6649d8f26732956d1c5424c82aae400a83d02b2cc2c87b:0 182264897 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:0 10000 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:1 182254756 2094512 No +Added a0c7f4ec9cccef2d89672a624a4e6c8237a17572efdd4679eea9e9ee70d2db04:0 10072679 2094513 Yes +Spent 25e0df5cc1aeb1b78e6056bf403e5e8b7e41f138060ca0a50a50134df0549a5e:2 540 2094508 No +Spent 42f383c04e09c26a2378272ec33aa0c1bf4883ca5ab739e8b7e06be5a5787d61:1 3848399 2007724 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:0 3788399 2094513 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:2 540 2094513 No +Spent a05880b8c77971ed0b9f73062c7c4cdb0ff3856ab14cbf8bc481ed571cd34b83:1 5591281046 2094511 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:0 5589696005 2094513 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:1 1565556 2094513 No +``` diff --git a/contrib/tracing/connectblock_benchmark.bt b/contrib/tracing/connectblock_benchmark.bt new file mode 100755 index 000000000000..6e7a98ef0766 --- /dev/null +++ b/contrib/tracing/connectblock_benchmark.bt @@ -0,0 +1,156 @@ +#!/usr/bin/env bpftrace + +/* + + USAGE: + + bpftrace contrib/tracing/connectblock_benchmark.bt + + - sets the height at which the benchmark should start. Setting + the start height to 0 starts the benchmark immediately, even before the + first block is connected. + - sets the height after which the benchmark should end. Setting + the end height to 0 disables the benchmark. The script only logs blocks + over . + - Threshold + + This script requires a 'bitcoind' binary compiled with eBPF support and the + 'validation:block_connected' USDT. By default, it's assumed that 'bitcoind' is + located in './src/bitcoind'. This can be modified in the script below. + + EXAMPLES: + + bpftrace contrib/tracing/connectblock_benchmark.bt 300000 680000 1000 + + When run together 'bitcoind -reindex', this benchmarks the time it takes to + connect the blocks between height 300.000 and 680.000 (inclusive) and prints + details about all blocks that take longer than 1000ms to connect. Prints a + histogram with block connection times when the benchmark is finished. + + + bpftrace contrib/tracing/connectblock_benchmark.bt 0 0 500 + + When running together 'bitcoind', all newly connected blocks that + take longer than 500ms to connect are logged. A histogram with block + connection times is shown when the script is terminated. + +*/ + +BEGIN +{ + $start_height = $1; + $end_height = $2; + $logging_threshold_ms = $3; + + if ($end_height < $start_height) { + printf("Error: start height (%d) larger than end height (%d)!\n", $start_height, $end_height); + exit(); + } + + if ($end_height > 0) { + printf("ConnectBlock benchmark between height %d and %d inclusive\n", $start_height, $end_height); + } else { + printf("ConnectBlock logging starting at height %d\n", $start_height); + } + + if ($logging_threshold_ms > 0) { + printf("Logging blocks taking longer than %d ms to connect.\n", $3); + } + + if ($start_height == 0) { + @start = nsecs; + } +} + +/* + Attaches to the 'validation:block_connected' USDT and collects stats when the + connected block is between the start and end height (or the end height is + unset). +*/ +usdt:./src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 || $2 == 0 )/ +{ + $height = arg1; + $transactions = arg2; + $inputs = arg3; + $sigops = arg4; + $duration = (uint64) arg5; + + @height = $height; + + @blocks = @blocks + 1; + @transactions = @transactions + $transactions; + @inputs = @inputs + $inputs; + @sigops = @sigops + $sigops; + + @durations = hist($duration / 1000); + + if ($height == $1 && $height != 0) { + @start = nsecs; + printf("Starting Connect Block Benchmark between height %d and %d.\n", $1, $2); + } + + if ($2 > 0 && $height >= $2) { + @end = nsecs; + $duration = @end - @start; + printf("\nTook %d ms to connect the blocks between height %d and %d.\n", $duration / 1000000, $1, $2); + exit(); + } +} + +/* + Attaches to the 'validation:block_connected' USDT and logs information about + blocks where the time it took to connect the block is above the + . +*/ +usdt:./src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / +{ + $hash = arg0; + $height = (int32) arg1; + $transactions = (uint64) arg2; + $inputs = (int32) arg3; + $sigops = (int64) arg4; + $duration = (int64) arg5; + + + printf("Block %d (", $height); + /* Prints each byte of the block hash as hex in big-endian (the block-explorer format) */ + $p = $hash + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p -= 1; + } + printf(") %4d tx %5d ins %5d sigops took %4d ms\n", $transactions, $inputs, $sigops, (uint64) $duration / 1000); +} + + +/* + Prints stats about the blocks, transactions, inputs, and sigops processed in + the last second (if any). +*/ +interval:s:1 { + if (@blocks > 0) { + printf("BENCH %4d blk/s %6d tx/s %7d inputs/s %8d sigops/s (height %d)\n", @blocks, @transactions, @inputs, @sigops, @height); + + zero(@blocks); + zero(@transactions); + zero(@inputs); + zero(@sigops); + } +} + +END +{ + printf("\nHistogram of block connection times in milliseconds (ms).\n"); + print(@durations); + + clear(@durations); + clear(@blocks); + clear(@transactions); + clear(@inputs); + clear(@sigops); + clear(@height); + clear(@start); + clear(@end); +} + diff --git a/contrib/tracing/log_p2p_traffic.bt b/contrib/tracing/log_p2p_traffic.bt new file mode 100755 index 000000000000..f62956aa5e1c --- /dev/null +++ b/contrib/tracing/log_p2p_traffic.bt @@ -0,0 +1,28 @@ +#!/usr/bin/env bpftrace + +BEGIN +{ + printf("Logging P2P traffic\n") +} + +usdt:./src/bitcoind:net:inbound_message +{ + $peer_id = (int64) arg0; + $peer_addr = str(arg1); + $peer_type = str(arg2); + $msg_type = str(arg3); + $msg_len = arg4; + printf("inbound '%s' msg from peer %d (%s, %s) with %d bytes\n", $msg_type, $peer_id, $peer_type, $peer_addr, $msg_len); +} + +usdt:./src/bitcoind:net:outbound_message +{ + $peer_id = (int64) arg0; + $peer_addr = str(arg1); + $peer_type = str(arg2); + $msg_type = str(arg3); + $msg_len = arg4; + + printf("outbound '%s' msg to peer %d (%s, %s) with %d bytes\n", $msg_type, $peer_id, $peer_type, $peer_addr, $msg_len); +} + diff --git a/contrib/tracing/log_raw_p2p_msgs.py b/contrib/tracing/log_raw_p2p_msgs.py new file mode 100755 index 000000000000..c0ab70410622 --- /dev/null +++ b/contrib/tracing/log_raw_p2p_msgs.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Demonstration of eBPF limitations and the effect on USDT with the + net:inbound_message and net:outbound_message tracepoints. """ + +# This script shows a limitation of eBPF when data larger than 32kb is passed to +# user-space. It uses BCC (https://github.com/iovisor/bcc) to load a sandboxed +# eBPF program into the Linux kernel (root privileges are required). The eBPF +# program attaches to two statically defined tracepoints. The tracepoint +# 'net:inbound_message' is called when a new P2P message is received, and +# 'net:outbound_message' is called on outbound P2P messages. The eBPF program +# submits the P2P messages to this script via a BPF ring buffer. The submitted +# messages are printed. + +# eBPF Limitations: +# +# Bitcoin P2P messages can be larger than 32kb (e.g. tx, block, ...). The eBPF +# VM's stack is limited to 512 bytes, and we can't allocate more than about 32kb +# for a P2P message in the eBPF VM. The message data is cut off when the message +# is larger than MAX_MSG_DATA_LENGTH (see definition below). This can be detected +# in user-space by comparing the data length to the message length variable. The +# message is cut off when the data length is smaller than the message length. +# A warning is included with the printed message data. +# +# Data is submitted to user-space (i.e. to this script) via a ring buffer. The +# throughput of the ring buffer is limited. Each p2p_message is about 32kb in +# size. In- or outbound messages submitted to the ring buffer in rapid +# succession fill the ring buffer faster than it can be read. Some messages are +# lost. +# +# BCC prints: "Possibly lost 2 samples" on lost messages. + +import sys +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +#include + +#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) + +// Maximum possible allocation size +// from include/linux/percpu.h in the Linux kernel +#define PCPU_MIN_UNIT_SIZE (32 << 10) + +// Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +#define MAX_PEER_ADDR_LENGTH 62 + 6 +#define MAX_PEER_CONN_TYPE_LENGTH 20 +#define MAX_MSG_TYPE_LENGTH 20 +#define MAX_MSG_DATA_LENGTH PCPU_MIN_UNIT_SIZE - 200 + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; + u8 msg[MAX_MSG_DATA_LENGTH]; +}; + +// We can't store the p2p_message struct on the eBPF stack as it is limited to +// 512 bytes and P2P message can be bigger than 512 bytes. However, we can use +// an BPF-array with a length of 1 to allocate up to 32768 bytes (this is +// defined by PCPU_MIN_UNIT_SIZE in include/linux/percpu.h in the Linux kernel). +// Also see https://github.com/iovisor/bcc/issues/2306 +BPF_ARRAY(msg_arr, struct p2p_message, 1); + +// Two BPF perf buffers for pushing data (here P2P messages) to user-space. +BPF_PERF_OUTPUT(inbound_messages); +BPF_PERF_OUTPUT(outbound_messages); + +int trace_inbound_message(struct pt_regs *ctx) { + int idx = 0; + struct p2p_message *msg = msg_arr.lookup(&idx); + + // lookup() does not return a NULL pointer. However, the BPF verifier + // requires an explicit check that that the `msg` pointer isn't a NULL + // pointer. See https://github.com/iovisor/bcc/issues/2595 + if (msg == NULL) return 1; + + bpf_usdt_readarg(1, ctx, &msg->peer_id); + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg->msg_size); + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); + + inbound_messages.perf_submit(ctx, msg, sizeof(*msg)); + return 0; +}; + +int trace_outbound_message(struct pt_regs *ctx) { + int idx = 0; + struct p2p_message *msg = msg_arr.lookup(&idx); + + // lookup() does not return a NULL pointer. However, the BPF verifier + // requires an explicit check that that the `msg` pointer isn't a NULL + // pointer. See https://github.com/iovisor/bcc/issues/2595 + if (msg == NULL) return 1; + + bpf_usdt_readarg(1, ctx, &msg->peer_id); + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg->msg_size); + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); + + outbound_messages.perf_submit(ctx, msg, sizeof(*msg)); + return 0; +}; +""" + + +def print_message(event, inbound): + print(f"%s %s msg '%s' from peer %d (%s, %s) with %d bytes: %s" % + ( + f"Warning: incomplete message (only %d out of %d bytes)!" % ( + len(event.msg), event.msg_size) if len(event.msg) < event.msg_size else "", + "inbound" if inbound else "outbound", + event.msg_type.decode("utf-8"), + event.peer_id, + event.peer_conn_type.decode("utf-8"), + event.peer_addr.decode("utf-8"), + event.msg_size, + bytes(event.msg[:event.msg_size]).hex(), + ) + ) + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="inbound_message", fn_name="trace_inbound_message") + bitcoind_with_usdts.enable_probe( + probe="outbound_message", fn_name="trace_outbound_message") + bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + # BCC: perf buffer handle function for inbound_messages + def handle_inbound(_, data, size): + """ Inbound message handler. + + Called each time a message is submitted to the inbound_messages BPF table.""" + + event = bpf["inbound_messages"].event(data) + print_message(event, True) + + # BCC: perf buffer handle function for outbound_messages + + def handle_outbound(_, data, size): + """ Outbound message handler. + + Called each time a message is submitted to the outbound_messages BPF table.""" + + event = bpf["outbound_messages"].event(data) + print_message(event, False) + + # BCC: add handlers to the inbound and outbound perf buffers + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + print("Logging raw P2P messages.") + print("Messages larger that about 32kb will be cut off!") + print("Some messages might be lost!") + while True: + try: + bpf.perf_buffer_poll() + except KeyboardInterrupt: + exit() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE:", sys.argv[0], "path/to/bitcoind") + exit() + path = sys.argv[1] + main(path) diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py new file mode 100755 index 000000000000..8c073bea0d18 --- /dev/null +++ b/contrib/tracing/log_utxocache_flush.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import sys +import ctypes +from bcc import BPF, USDT + +"""Example logging Bitcoin Core utxo set cache flushes utilizing + the utxocache:flush tracepoint.""" + +# USAGE: ./contrib/tracing/log_utxocache_flush.py path/to/bitcoind + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +# include + +struct data_t +{ + u64 duration; + u32 mode; + u64 coins_count; + u64 coins_mem_usage; + bool is_flush_for_prune; +}; + +// BPF perf buffer to push the data to user space. +BPF_PERF_OUTPUT(flush); + +int trace_flush(struct pt_regs *ctx) { + struct data_t data = {}; + bpf_usdt_readarg(1, ctx, &data.duration); + bpf_usdt_readarg(2, ctx, &data.mode); + bpf_usdt_readarg(3, ctx, &data.coins_count); + bpf_usdt_readarg(4, ctx, &data.coins_mem_usage); + bpf_usdt_readarg(5, ctx, &data.is_flush_for_prune); + flush.perf_submit(ctx, &data, sizeof(data)); + return 0; +} +""" + +FLUSH_MODES = [ + 'NONE', + 'IF_NEEDED', + 'PERIODIC', + 'ALWAYS' +] + + +class Data(ctypes.Structure): + # define output data structure corresponding to struct data_t + _fields_ = [ + ("duration", ctypes.c_uint64), + ("mode", ctypes.c_uint32), + ("coins_count", ctypes.c_uint64), + ("coins_mem_usage", ctypes.c_uint64), + ("is_flush_for_prune", ctypes.c_bool) + ] + + +def print_event(event): + print("%-15d %-10s %-15d %-15s %-8s" % ( + event.duration, + FLUSH_MODES[event.mode], + event.coins_count, + "%.2f kB" % (event.coins_mem_usage/1000), + event.is_flush_for_prune + )) + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program + # to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="flush", fn_name="trace_flush") + b = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + def handle_flush(_, data, size): + """ Coins Flush handler. + Called each time coin caches and indexes are flushed.""" + event = ctypes.cast(data, ctypes.POINTER(Data)).contents + print_event(event) + + b["flush"].open_perf_buffer(handle_flush) + print("Logging utxocache flushes. Ctrl-C to end...") + print("%-15s %-10s %-15s %-15s %-8s" % ("Duration (µs)", "Mode", + "Coins Count", "Memory Usage", + "Flush for Prune")) + + while True: + try: + b.perf_buffer_poll() + except KeyboardInterrupt: + exit(0) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE: ", sys.argv[0], "path/to/bitcoind") + exit(1) + + path = sys.argv[1] + main(path) diff --git a/contrib/tracing/log_utxos.bt b/contrib/tracing/log_utxos.bt new file mode 100755 index 000000000000..54d5010f825e --- /dev/null +++ b/contrib/tracing/log_utxos.bt @@ -0,0 +1,86 @@ +#!/usr/bin/env bpftrace + +/* + + USAGE: + + bpftrace contrib/tracing/log_utxos.bt + + This script requires a 'bitcoind' binary compiled with eBPF support and the + 'utxocache' tracepoints. By default, it's assumed that 'bitcoind' is + located in './src/bitcoind'. This can be modified in the script below. + + NOTE: requires bpftrace v0.12.0 or above. +*/ + +BEGIN +{ + printf("%-7s %-71s %16s %7s %8s\n", + "OP", "Outpoint", "Value", "Height", "Coinbase"); +} + +/* + Attaches to the 'utxocache:add' tracepoint and prints additions to the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:add +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Added "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:spent' tracepoint and prints spents from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:spent +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Spent "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:uncache' tracepoint and uncache UTXOs from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:uncache +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Uncache "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} diff --git a/contrib/tracing/p2p_monitor.py b/contrib/tracing/p2p_monitor.py new file mode 100755 index 000000000000..4ff701cac3c8 --- /dev/null +++ b/contrib/tracing/p2p_monitor.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Interactive bitcoind P2P network traffic monitor utilizing USDT and the + net:inbound_message and net:outbound_message tracepoints. """ + +# This script demonstrates what USDT for Bitcoin Core can enable. It uses BCC +# (https://github.com/iovisor/bcc) to load a sandboxed eBPF program into the +# Linux kernel (root privileges are required). The eBPF program attaches to two +# statically defined tracepoints. The tracepoint 'net:inbound_message' is called +# when a new P2P message is received, and 'net:outbound_message' is called on +# outbound P2P messages. The eBPF program submits the P2P messages to +# this script via a BPF ring buffer. + +import sys +import curses +from curses import wrapper, panel +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +#include + +// Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +// I2P addresses are 60 chars + 6 chars for the port (':12345'). +#define MAX_PEER_ADDR_LENGTH 62 + 6 +#define MAX_PEER_CONN_TYPE_LENGTH 20 +#define MAX_MSG_TYPE_LENGTH 20 + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; +}; + + +// Two BPF perf buffers for pushing data (here P2P messages) to user space. +BPF_PERF_OUTPUT(inbound_messages); +BPF_PERF_OUTPUT(outbound_messages); + +int trace_inbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + + inbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; + +int trace_outbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + + outbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; +""" + + +class Message: + """ A P2P network message. """ + msg_type = "" + size = 0 + data = bytes() + inbound = False + + def __init__(self, msg_type, size, inbound): + self.msg_type = msg_type + self.size = size + self.inbound = inbound + + +class Peer: + """ A P2P network peer. """ + id = 0 + address = "" + connection_type = "" + last_messages = list() + + total_inbound_msgs = 0 + total_inbound_bytes = 0 + total_outbound_msgs = 0 + total_outbound_bytes = 0 + + def __init__(self, id, address, connection_type): + self.id = id + self.address = address + self.connection_type = connection_type + self.last_messages = list() + + def add_message(self, message): + self.last_messages.append(message) + if len(self.last_messages) > 25: + self.last_messages.pop(0) + if message.inbound: + self.total_inbound_bytes += message.size + self.total_inbound_msgs += 1 + else: + self.total_outbound_bytes += message.size + self.total_outbound_msgs += 1 + + +def main(bitcoind_path): + peers = dict() + + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="inbound_message", fn_name="trace_inbound_message") + bitcoind_with_usdts.enable_probe( + probe="outbound_message", fn_name="trace_outbound_message") + bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + # BCC: perf buffer handle function for inbound_messages + def handle_inbound(_, data, size): + """ Inbound message handler. + + Called each time a message is submitted to the inbound_messages BPF table.""" + event = bpf["inbound_messages"].event(data) + if event.peer_id not in peers: + peer = Peer(event.peer_id, event.peer_addr.decode( + "utf-8"), event.peer_conn_type.decode("utf-8")) + peers[peer.id] = peer + peers[event.peer_id].add_message( + Message(event.msg_type.decode("utf-8"), event.msg_size, True)) + + # BCC: perf buffer handle function for outbound_messages + def handle_outbound(_, data, size): + """ Outbound message handler. + + Called each time a message is submitted to the outbound_messages BPF table.""" + event = bpf["outbound_messages"].event(data) + if event.peer_id not in peers: + peer = Peer(event.peer_id, event.peer_addr.decode( + "utf-8"), event.peer_conn_type.decode("utf-8")) + peers[peer.id] = peer + peers[event.peer_id].add_message( + Message(event.msg_type.decode("utf-8"), event.msg_size, False)) + + # BCC: add handlers to the inbound and outbound perf buffers + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + wrapper(loop, bpf, peers) + + +def loop(screen, bpf, peers): + screen.nodelay(1) + cur_list_pos = 0 + win = curses.newwin(30, 70, 2, 7) + win.erase() + win.border(ord("|"), ord("|"), ord("-"), ord("-"), + ord("-"), ord("-"), ord("-"), ord("-")) + info_panel = panel.new_panel(win) + info_panel.hide() + + ROWS_AVALIABLE_FOR_LIST = curses.LINES - 5 + scroll = 0 + + while True: + try: + # BCC: poll the perf buffers for new events or timeout after 50ms + bpf.perf_buffer_poll(timeout=50) + + ch = screen.getch() + if (ch == curses.KEY_DOWN or ch == ord("j")) and cur_list_pos < len( + peers.keys()) -1 and info_panel.hidden(): + cur_list_pos += 1 + if cur_list_pos >= ROWS_AVALIABLE_FOR_LIST: + scroll += 1 + if (ch == curses.KEY_UP or ch == ord("k")) and cur_list_pos > 0 and info_panel.hidden(): + cur_list_pos -= 1 + if scroll > 0: + scroll -= 1 + if ch == ord('\n') or ch == ord(' '): + if info_panel.hidden(): + info_panel.show() + else: + info_panel.hide() + screen.erase() + render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel) + curses.panel.update_panels() + screen.refresh() + except KeyboardInterrupt: + exit() + + +def render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel): + """ renders the list of peers and details panel + + This code is unrelated to USDT, BCC and BPF. + """ + header_format = "%6s %-20s %-20s %-22s %-67s" + row_format = "%6s %-5d %9d byte %-5d %9d byte %-22s %-67s" + + screen.addstr(0, 1, (" P2P Message Monitor "), curses.A_REVERSE) + screen.addstr( + 1, 0, (" Navigate with UP/DOWN or J/K and select a peer with ENTER or SPACE to see individual P2P messages"), curses.A_NORMAL) + screen.addstr(3, 0, + header_format % ("PEER", "OUTBOUND", "INBOUND", "TYPE", "ADDR"), curses.A_BOLD | curses.A_UNDERLINE) + peer_list = sorted(peers.keys())[scroll:ROWS_AVALIABLE_FOR_LIST+scroll] + for i, peer_id in enumerate(peer_list): + peer = peers[peer_id] + screen.addstr(i + 4, 0, + row_format % (peer.id, peer.total_outbound_msgs, peer.total_outbound_bytes, + peer.total_inbound_msgs, peer.total_inbound_bytes, + peer.connection_type, peer.address), + curses.A_REVERSE if i + scroll == cur_list_pos else curses.A_NORMAL) + if i + scroll == cur_list_pos: + info_window = info_panel.window() + info_window.erase() + info_window.border( + ord("|"), ord("|"), ord("-"), ord("-"), + ord("-"), ord("-"), ord("-"), ord("-")) + + info_window.addstr( + 1, 1, f"PEER {peer.id} ({peer.address})".center(68), curses.A_REVERSE | curses.A_BOLD) + info_window.addstr( + 2, 1, f" OUR NODE{peer.connection_type:^54}PEER ", + curses.A_BOLD) + for i, msg in enumerate(peer.last_messages): + if msg.inbound: + info_window.addstr( + i + 3, 1, "%68s" % + (f"<--- {msg.msg_type} ({msg.size} bytes) "), curses.A_NORMAL) + else: + info_window.addstr( + i + 3, 1, " %s (%d byte) --->" % + (msg.msg_type, msg.size), curses.A_NORMAL) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE:", sys.argv[0], "path/to/bitcoind") + exit() + path = sys.argv[1] + main(path) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index ece02dc24e8f..d6856b42749e 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -13,8 +13,8 @@ # # Note that suppressions may depend on OS and/or library versions. # Tested on: -# * aarch64 (Ubuntu 20.04 system libs, without gui) -# * x86_64 (Ubuntu 18.04 system libs, without gui) +# * aarch64 (Ubuntu 22.04 system libs, clang, without gui) +# * x86_64 (Ubuntu 22.04 system libs, clang, without gui) { Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 Memcheck:Leak @@ -64,12 +64,6 @@ ... obj:*/libdb_cxx-*.so } -{ - Suppress leaks on init - Memcheck:Leak - ... - fun:_Z11AppInitMainR11NodeContext -} { Suppress leaks on shutdown Memcheck:Leak @@ -82,21 +76,6 @@ ... obj:/usr/lib64/libgdk-3.so.0.2404.7 } -{ - Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - fun:_ZN7leveldbL10InitModuleEv -} -{ - Suppress leveldb warning (leveldb::Env::Default()) - https://github.com/google/leveldb/issues/113 - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - ... - fun:_ZN7leveldbL14InitDefaultEnvEv -} { Suppress leveldb leak Memcheck:Leak @@ -112,54 +91,6 @@ ... fun:GetCoin } -{ - Suppress wcsnrtombs glibc SSE4 warning (could be related: https://stroika.atlassian.net/browse/STK-626) - Memcheck:Addr16 - fun:__wcsnlen_sse4_1 - fun:wcsnrtombs -} -{ - Suppress wcsnrtombs warning (remove after removing boost::fs) - Memcheck:Cond - ... - fun:_ZN5boost10filesystem6detail11unique_pathERKNS0_4pathEPNS_6system10error_codeE -} -{ - Suppress boost warning - Memcheck:Leak - fun:_Znwm - ... - fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE - fun:_ZN5boost9unit_test9framework3runEmb - fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc - fun:main -} -{ - Suppress boost::filesystem warning (fixed in boost 1.70: https://github.com/boostorg/filesystem/commit/bbe9d1771e5d679b3f10c42a58fc81f7e8c024a9) - Memcheck:Cond - fun:_ZN5boost10filesystem6detail28directory_iterator_incrementERNS0_18directory_iteratorEPNS_6system10error_codeE - ... - obj:*/libboost_filesystem.so.* -} -{ - Suppress boost::filesystem warning (could be related: https://stackoverflow.com/questions/9830182/function-boostfilesystemcomplete-being-reported-as-possible-memory-leak-by-v) - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - ... - fun:_ZN5boost10filesystem8absoluteERKNS0_4pathES3_ -} -{ - Suppress boost still reachable memory warning - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - ... - fun:_M_construct_aux - fun:_M_construct - fun:basic_string - fun:path -} { Suppress LogInstance still reachable memory warning Memcheck:Leak diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index e95a57586ff1..b8b15280bae2 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -40,7 +40,7 @@ Import trusted keys In order to check the commit signatures, you must add the trusted PGP keys to your machine. [GnuPG](https://gnupg.org/) may be used to import the trusted keys by running the following command: ```sh -gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys $( /dev/null 2>&1; then + if ! ./contrib/verify-commits/verify-commits.py "$3" > /dev/null 2>&1; then echo "ERROR: A commit is not signed, can't push" ./contrib/verify-commits/verify-commits.py exit 1 diff --git a/contrib/verify-commits/trusted-git-root b/contrib/verify-commits/trusted-git-root index c60f8ab695e9..1c421959618e 100644 --- a/contrib/verify-commits/trusted-git-root +++ b/contrib/verify-commits/trusted-git-root @@ -1 +1 @@ -82bcf405f6db1d55b684a1f63a4aabad376cdad7 +577bd51a4b8de066466a445192c1c653872657e2 diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index c14f90b04b71..5ca65e7b0d18 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -1,7 +1,6 @@ 71A3B16735405025D447E8F274810B012346C9A6 -133EAC179436F14A5CF1B794860FEB804E669320 -32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B -CA03882CB1FC067B5D3ACFE4D300116E1C875A3D E777299FC265DD04793070EB944D35F9AC3DB76A D1DBF2C4B96F2DEBF4C16654410108112E7EA81F +152812300785C96444D3334D17565732E08E5E41 +6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 7e46c6fd4773..2ff14c1f86d6 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -82,11 +82,16 @@ def main(): # get directory of this program and read data files dirname = os.path.dirname(os.path.abspath(__file__)) print("Using verify-commits data from " + dirname) - verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0] - verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0] - revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines() - unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines() - incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines() + with open(dirname + "/trusted-git-root", "r", encoding="utf8") as f: + verified_root = f.read().splitlines()[0] + with open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8") as f: + verified_sha512_root = f.read().splitlines()[0] + with open(dirname + "/allow-revsig-commits", "r", encoding="utf8") as f: + revsig_allowed = f.read().splitlines() + with open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf8") as f: + unclean_merge_allowed = f.read().splitlines() + with open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf8") as f: + incorrect_sha512_allowed = f.read().splitlines() # Set commit and branch and set variables current_commit = args.commit diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 6cbaf2dc0c7c..b5e4f1318b95 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Script for verifying Bitoin Core release binaries +"""Script for verifying Bitcoin Core release binaries This script attempts to download the signature file SHA256SUMS.asc from bitcoincore.org and bitcoin.org and compares them. diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh index 29802e622ed1..82fcf2d40684 100755 --- a/contrib/windeploy/detached-sig-create.sh +++ b/contrib/windeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,6 +23,7 @@ TIMESERVER=http://timestamp.comodoca.com CERTFILE="win-codesign.cert" mkdir -p "${OUTSUBDIR}" +# shellcheck disable=SC2046 basename -a $(ls -1 "${SRCDIR}"/*-unsigned.exe) | while read UNSIGNED; do echo Signing "${UNSIGNED}" "${OSSLSIGNCODE}" sign -certs "${CERTFILE}" -t "${TIMESERVER}" -h sha256 -in "${SRCDIR}/${UNSIGNED}" -out "${WORKDIR}/${UNSIGNED}" "$@" diff --git a/contrib/windeploy/win-codesign.cert b/contrib/windeploy/win-codesign.cert index e763df584787..22f17296b6aa 100644 --- a/contrib/windeploy/win-codesign.cert +++ b/contrib/windeploy/win-codesign.cert @@ -1,89 +1,112 @@ +-----BEGIN CERTIFICATE----- +MIIHfDCCBWSgAwIBAgIQCmVvdQal72U2QxbUTT3SRTANBgkqhkiG9w0BAQsFADBp +MQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xQTA/BgNVBAMT +OERpZ2lDZXJ0IFRydXN0ZWQgRzQgQ29kZSBTaWduaW5nIFJTQTQwOTYgU0hBMzg0 +IDIwMjEgQ0ExMB4XDTIyMDUyNDAwMDAwMFoXDTI0MDUyOTIzNTk1OVowgYAxCzAJ +BgNVBAYTAlVTMREwDwYDVQQIEwhEZWxhd2FyZTEOMAwGA1UEBxMFTGV3ZXMxJjAk +BgNVBAoTHUJpdGNvaW4gQ29yZSBDb2RlIFNpZ25pbmcgTExDMSYwJAYDVQQDEx1C +aXRjb2luIENvcmUgQ29kZSBTaWduaW5nIExMQzCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBALewxfjztuRTDNAGf7zkqqWNEt28CZmVJHoYltVRxtE1BP45 +BfmptH5eM1JC/XosTPytHRFeOkO4YVAtiELxK9S/82OZlKA7Mx7PW6vv1184u8+m +P3WpTN/KAZTaW9fB0ELTSCuqsvXq2crM2T7NudJnSyWh2VBjLfPPCAcYwzyGKQbl +jQWjFEJDJWFK83t9mK/v0WQgA3jGJeaz+V6CYXMS7UgpdG8dUhg9o63gYJZAW5pY +RIsNRcJCM5LHhwEMW5329UsTmYCfP7/53emepbQ0n8ijVZjgaJ+LZ8NspBLSeCiF +9UPCKX82uWiQAUTbYHCfSi3I0f3wQidXL9ZY+PXmalM7BMuQ+c2xEcl97CnhrDzx +EBwZvvOC9wGoG+8+epV4TjUZWf+7QN1ZYeg1rai7c7c8u9ILogE8su2xVoz333TH +CDvScIgnQXmk+cbKMBtg9kM0F+aLWsN2xVf0uAj3U7sdXLrfJeW0DZIktWtTBQzX +O/OE4Ka+1WFnDg0HJIih0cTjl9YYvfe53L4pCGy+qGt/XGBRqCMfXp3g+H9FGR5r +pensVVcsrv3GbTfYdlpdmp9OHH5G57GTAZueobCZg7r7RKK0zPU9EiTLJxzyXuai +v/Ksd8eIhHRjewMaQuAtQM1tO+oKAbLF0v2M7v7/aVT76X32JllYAizm3zjvAgMB +AAGjggIGMIICAjAfBgNVHSMEGDAWgBRoN+Drtjv4XxGG+/5hewiIZfROQjAdBgNV +HQ4EFgQUvCpU58PIuofv0kHJ3Ty0YDKEy3cwDgYDVR0PAQH/BAQDAgeAMBMGA1Ud +JQQMMAoGCCsGAQUFBwMDMIG1BgNVHR8Ega0wgaowU6BRoE+GTWh0dHA6Ly9jcmwz +LmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNENvZGVTaWduaW5nUlNBNDA5 +NlNIQTM4NDIwMjFDQTEuY3JsMFOgUaBPhk1odHRwOi8vY3JsNC5kaWdpY2VydC5j +b20vRGlnaUNlcnRUcnVzdGVkRzRDb2RlU2lnbmluZ1JTQTQwOTZTSEEzODQyMDIx +Q0ExLmNybDA+BgNVHSAENzA1MDMGBmeBDAEEATApMCcGCCsGAQUFBwIBFhtodHRw +Oi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwgZQGCCsGAQUFBwEBBIGHMIGEMCQGCCsG +AQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wXAYIKwYBBQUHMAKGUGh0 +dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNENvZGVT +aWduaW5nUlNBNDA5NlNIQTM4NDIwMjFDQTEuY3J0MAwGA1UdEwEB/wQCMAAwDQYJ +KoZIhvcNAQELBQADggIBABhpTZufRws1vrtI0xB1/UWrSEJxdPHivfpXE708dzum +Jh3TFzpsEUCQX5BJJet1l7x92sKNeAL7votA+8O8YvMD64Kim7VKA2BB8AOHKQbp +r1c2iZBwwofInviRYvsrvQta6KBy2KOe1L/l0KnpUazL9Tv4VKvuWAw/Qc0/eTQr +NZRsmADORxnZ1qW+SpF+/WbazIYjod/Oqb1U3on+PzyiGD3SjzNhsdFRptqzrIaY +UVV+2XHG4fN6A8wkyQL5NIVXGiK7rqS5VrRAv58Lf1ZZTghdAL+5SySE0OsR9t0K +W73ZB9pxbuZZ6Zfxjotjw+IilCEm3ADbc7Eb2ijI4x8mix0XWMUrhL34s7/jRyDi +P+30aSgjWp611tp/EYRW5kpIaFR8AesDdM0DSSCCRXOMwQG2Tq2+CnqItB5oLNPp +2XySwlIWvmjbzsREfIpE3yh3bxmHY+vFIc2R0nNkbWNIT6AGtaEQ7oWkgpK8YMkA +QCf4EUC4Qa7qHiH6YSmYJhjApBLC7UDwevgwxuDrwimWAj+tDkzdnENMcBp4SAy6 +LwUuDi2IU6HRSXWdh2YEkDbc3FdwknnnEWaB4dlRL85YjHyLXN0KiE7SKTj1LfR4 +dGeDqVUlDj9D5+X4a7F89wLP/um40/52HUQv5t5WcNr/47r9aVkx9DHs1b8oUnLg +-----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIGQzCCBSugAwIBAgIQBSN7Cm16Z0UT9p7lA2jiKDANBgkqhkiG9w0BAQsFADBy +MIIGsDCCBJigAwIBAgIQCK1AsmDSnEyfXs2pvZOu2TANBgkqhkiG9w0BAQwFADBi MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMTEwLwYDVQQDEyhEaWdpQ2VydCBTSEEyIEFzc3VyZWQg -SUQgQ29kZSBTaWduaW5nIENBMB4XDTIxMDUyMTAwMDAwMFoXDTIyMDUyNjIzNTk1 -OVowgYAxCzAJBgNVBAYTAlVTMREwDwYDVQQIEwhEZWxhd2FyZTEOMAwGA1UEBxMF -TGV3ZXMxJjAkBgNVBAoTHUJpdGNvaW4gQ29yZSBDb2RlIFNpZ25pbmcgTExDMSYw -JAYDVQQDEx1CaXRjb2luIENvcmUgQ29kZSBTaWduaW5nIExMQzCCAiIwDQYJKoZI -hvcNAQEBBQADggIPADCCAgoCggIBAKe6xtFgKAQ68MvxwCjNtpgPobfDQCLKvCAN -uBKGYuub6ufQB5dhCLN9fjMgfg33AyauvU3PcEUDUWD3/k925bPqgxHC3E7YqoB+ -11b/2Y7a86okqUgcGgvKhaKoHmXxElpM9EjQHjJ0yL4QAR1Lp+9CMMW3wIulBYKt -wLIArFvbuQhMO/6rxL8frpK049v//WfQzB16GXuFnzN/6fDK7oOt5IrKTg4H6EY2 -fj4+QaUj0lNX7aHnZ6Ki45h2RUPDgN1ipRIuhM67npyZ/tdzPPjI3PUgfXCccN6D -+qWWnbbbvPuOht4ziPciVnPd57PqJmAOnLI86gisDfd7VKlcpOSEaagdUGvMbU6f -uAps818GwnJzwCGllxlKASCgXDAckLLvMuit4RfYAhhdhw5R0AsaWK0HW88oHOqi -U7eWlMCbSGk34x9hBrxYl7tvcNcLPWIPYrrhFWNFpkV8bVVIoV5rUNRgWvBcdOq1 -CCPTfsJp3nEH2WCoBghZquDZLSW12wMw2UsQyEojBeGhrR1inn8uK93wSnVCC8F4 -21yWNRMNe/LQVhmZDgFOen9r/WijBsBdQw1bL8N4zGdYv8+soqkrWzW417FfSx81 -pj4j5FEXYXXV5k/4/eBpIARXVRR8xya0nGkhNJmBk0jjDGD8fPW2gFQbqnUwAQ34 -vOr8NUqHAgMBAAGjggHEMIIBwDAfBgNVHSMEGDAWgBRaxLl7KgqjpepxA8Bg+S32 -ZXUOWDAdBgNVHQ4EFgQUVSLtZnifEHvd8z3E7AyLYNuDiaMwDgYDVR0PAQH/BAQD -AgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMHcGA1UdHwRwMG4wNaAzoDGGL2h0dHA6 -Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9zaGEyLWFzc3VyZWQtY3MtZzEuY3JsMDWgM6Ax -hi9odHRwOi8vY3JsNC5kaWdpY2VydC5jb20vc2hhMi1hc3N1cmVkLWNzLWcxLmNy -bDBLBgNVHSAERDBCMDYGCWCGSAGG/WwDATApMCcGCCsGAQUFBwIBFhtodHRwOi8v -d3d3LmRpZ2ljZXJ0LmNvbS9DUFMwCAYGZ4EMAQQBMIGEBggrBgEFBQcBAQR4MHYw -JAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBOBggrBgEFBQcw -AoZCaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0U0hBMkFzc3Vy -ZWRJRENvZGVTaWduaW5nQ0EuY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQEL -BQADggEBAOaJneI91NJgqghUxgc0AWQ01SAJTgN4z7xMQ3W0ZAtwGbA0byT7YRlj -j7h+j+hMX/JYkRJETTh8Nalq2tPWJBiMMEPOGFVttFER1pwouHkK9pSKyp4xRvNU -L0LPh7fE4EYMJoynys6ZTpMCHLku+X3jFat1+1moh9TJRvK5+ETZYGl0seFNU3mJ -dZzusObm4scffIGgi40kmmISKd5ZRuooRTu9FFR/3vpfbA+7Vg4RSH3CcQPo9bfk -+h/qRQhSfQInTBn7obRpIlvEcK782qivqseJGdtnTmcdVRShD5ckTVza1yv25uQz -l/yTqmG2LXlYjl5iMSdF0C1xYq6IsOA= +d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg +RzQwHhcNMjEwNDI5MDAwMDAwWhcNMzYwNDI4MjM1OTU5WjBpMQswCQYDVQQGEwJV +UzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xQTA/BgNVBAMTOERpZ2lDZXJ0IFRy +dXN0ZWQgRzQgQ29kZSBTaWduaW5nIFJTQTQwOTYgU0hBMzg0IDIwMjEgQ0ExMIIC +IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1bQvQtAorXi3XdU5WRuxiEL1 +M4zrPYGXcMW7xIUmMJ+kjmjYXPXrNCQH4UtP03hD9BfXHtr50tVnGlJPDqFX/IiZ +wZHMgQM+TXAkZLON4gh9NH1MgFcSa0OamfLFOx/y78tHWhOmTLMBICXzENOLsvsI +8IrgnQnAZaf6mIBJNYc9URnokCF4RS6hnyzhGMIazMXuk0lwQjKP+8bqHPNlaJGi +TUyCEUhSaN4QvRRXXegYE2XFf7JPhSxIpFaENdb5LpyqABXRN/4aBpTCfMjqGzLm +ysL0p6MDDnSlrzm2q2AS4+jWufcx4dyt5Big2MEjR0ezoQ9uo6ttmAaDG7dqZy3S +vUQakhCBj7A7CdfHmzJawv9qYFSLScGT7eG0XOBv6yb5jNWy+TgQ5urOkfW+0/tv +k2E0XLyTRSiDNipmKF+wc86LJiUGsoPUXPYVGUztYuBeM/Lo6OwKp7ADK5GyNnm+ +960IHnWmZcy740hQ83eRGv7bUKJGyGFYmPV8AhY8gyitOYbs1LcNU9D4R+Z1MI3s +MJN2FKZbS110YU0/EpF23r9Yy3IQKUHw1cVtJnZoEUETWJrcJisB9IlNWdt4z4FK +PkBHX8mBUHOFECMhWWCKZFTBzCEa6DgZfGYczXg4RTCZT/9jT0y7qg0IU0F8WD1H +s/q27IwyCQLMbDwMVhECAwEAAaOCAVkwggFVMBIGA1UdEwEB/wQIMAYBAf8CAQAw +HQYDVR0OBBYEFGg34Ou2O/hfEYb7/mF7CIhl9E5CMB8GA1UdIwQYMBaAFOzX44LS +cV1kTN8uZz/nupiuHA9PMA4GA1UdDwEB/wQEAwIBhjATBgNVHSUEDDAKBggrBgEF +BQcDAzB3BggrBgEFBQcBAQRrMGkwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRp +Z2ljZXJ0LmNvbTBBBggrBgEFBQcwAoY1aHR0cDovL2NhY2VydHMuZGlnaWNlcnQu +Y29tL0RpZ2lDZXJ0VHJ1c3RlZFJvb3RHNC5jcnQwQwYDVR0fBDwwOjA4oDagNIYy +aHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0VHJ1c3RlZFJvb3RHNC5j +cmwwHAYDVR0gBBUwEzAHBgVngQwBAzAIBgZngQwBBAEwDQYJKoZIhvcNAQEMBQAD +ggIBADojRD2NCHbuj7w6mdNW4AIapfhINPMstuZ0ZveUcrEAyq9sMCcTEp6QRJ9L +/Z6jfCbVN7w6XUhtldU/SfQnuxaBRVD9nL22heB2fjdxyyL3WqqQz/WTauPrINHV +UHmImoqKwba9oUgYftzYgBoRGRjNYZmBVvbJ43bnxOQbX0P4PpT/djk9ntSZz0rd +KOtfJqGVWEjVGv7XJz/9kNF2ht0csGBc8w2o7uCJob054ThO2m67Np375SFTWsPK +6Wrxoj7bQ7gzyE84FJKZ9d3OVG3ZXQIUH0AzfAPilbLCIXVzUstG2MQ0HKKlS43N +b3Y3LIU/Gs4m6Ri+kAewQ3+ViCCCcPDMyu/9KTVcH4k4Vfc3iosJocsL6TEa/y4Z +XDlx4b6cpwoG1iZnt5LmTl/eeqxJzy6kdJKt2zyknIYf48FWGysj/4+16oh7cGvm +oLr9Oj9FpsToFpFSi0HASIRLlk2rREDjjfAVKM7t8RhWByovEMQMCGQ8M4+uKIw8 +y4+ICw2/O/TOHnuO77Xry7fwdxPm5yg/rBKupS8ibEH5glwVZsxsDsrFhsP2JjMM +B0ug0wcCampAMEhLNKhRILutG4UI4lkNbcoFUCvqShyepf2gpx8GdOfy1lKQ/a+F +SCH5Vzu0nAPthkX0tGFuv2jiJmCG6sivqf6UHedjGzqGVnhO -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIFMDCCBBigAwIBAgIQBAkYG1/Vu2Z1U0O1b5VQCDANBgkqhkiG9w0BAQsFADBl +MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv -b3QgQ0EwHhcNMTMxMDIyMTIwMDAwWhcNMjgxMDIyMTIwMDAwWjByMQswCQYDVQQG -EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl -cnQuY29tMTEwLwYDVQQDEyhEaWdpQ2VydCBTSEEyIEFzc3VyZWQgSUQgQ29kZSBT -aWduaW5nIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA+NOzHH8O -Ea9ndwfTCzFJGc/Q+0WZsTrbRPV/5aid2zLXcep2nQUut4/6kkPApfmJ1DcZ17aq -8JyGpdglrA55KDp+6dFn08b7KSfH03sjlOSRI5aQd4L5oYQjZhJUM1B0sSgmuyRp -wsJS8hRniolF1C2ho+mILCCVrhxKhwjfDPXiTWAYvqrEsq5wMWYzcT6scKKrzn/p -fMuSoeU7MRzP6vIK5Fe7SrXpdOYr/mzLfnQ5Ng2Q7+S1TqSp6moKq4TzrGdOtcT3 -jNEgJSPrCGQ+UpbB8g8S9MWOD8Gi6CxR93O8vYWxYoNzQYIH5DiLanMg0A9kczye -n6Yzqf0Z3yWT0QIDAQABo4IBzTCCAckwEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNV -HQ8BAf8EBAMCAYYwEwYDVR0lBAwwCgYIKwYBBQUHAwMweQYIKwYBBQUHAQEEbTBr -MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wQwYIKwYBBQUH -MAKGN2h0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydEFzc3VyZWRJ -RFJvb3RDQS5jcnQwgYEGA1UdHwR6MHgwOqA4oDaGNGh0dHA6Ly9jcmw0LmRpZ2lj -ZXJ0LmNvbS9EaWdpQ2VydEFzc3VyZWRJRFJvb3RDQS5jcmwwOqA4oDaGNGh0dHA6 -Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydEFzc3VyZWRJRFJvb3RDQS5jcmww -TwYDVR0gBEgwRjA4BgpghkgBhv1sAAIEMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8v -d3d3LmRpZ2ljZXJ0LmNvbS9DUFMwCgYIYIZIAYb9bAMwHQYDVR0OBBYEFFrEuXsq -CqOl6nEDwGD5LfZldQ5YMB8GA1UdIwQYMBaAFEXroq/0ksuCMS1Ri6enIZ3zbcgP -MA0GCSqGSIb3DQEBCwUAA4IBAQA+7A1aJLPzItEVyCx8JSl2qB1dHC06GsTvMGHX -fgtg/cM9D8Svi/3vKt8gVTew4fbRknUPUbRupY5a4l4kgU4QpO4/cY5jDhNLrddf -RHnzNhQGivecRk5c/5CxGwcOkRX7uq+1UcKNJK4kxscnKqEpKBo6cSgCPC6Ro8Al -EeKcFEehemhor5unXCBc2XGxDI+7qPjFEmifz0DLQESlE/DmZAwlCEIysjaKJAL+ -L3J+HNdJRZboWR3p+nRka7LrZkPas7CM1ekN3fYBIM6ZMWM9CBoYs4GbT8aTEAb8 -B4H6i9r5gkn3Ym6hU/oSlBiFLpKR6mhsRDKyZqHnGKSaZFHv +d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg +RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu +Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y +ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If +xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV +ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO +DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ +jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/ +CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi +EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM +fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY +uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK +chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t +9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD +ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2 +SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd ++SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc +fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa +sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N +cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N +0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie +4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI +r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1 +/YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm +gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+ -----END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIDtzCCAp+gAwIBAgIQDOfg5RfYRv6P5WD8G/AwOTANBgkqhkiG9w0BAQUFADBl -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv -b3QgQ0EwHhcNMDYxMTEwMDAwMDAwWhcNMzExMTEwMDAwMDAwWjBlMQswCQYDVQQG -EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl -cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgQ0EwggEi -MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtDhXO5EOAXLGH87dg+XESpa7c -JpSIqvTO9SA5KFhgDPiA2qkVlTJhPLWxKISKityfCgyDF3qPkKyK53lTXDGEKvYP -mDI2dsze3Tyoou9q+yHyUmHfnyDXH+Kx2f4YZNISW1/5WBg1vEfNoTb5a3/UsDg+ -wRvDjDPZ2C8Y/igPs6eD1sNuRMBhNZYW/lmci3Zt1/GiSw0r/wty2p5g0I6QNcZ4 -VYcgoc/lbQrISXwxmDNsIumH0DJaoroTghHtORedmTpyoeb6pNnVFzF1roV9Iq4/ -AUaG9ih5yLHa5FcXxH4cDrC0kqZWs72yl+2qp/C3xag/lRbQ/6GW6whfGHdPAgMB -AAGjYzBhMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW -BBRF66Kv9JLLgjEtUYunpyGd823IDzAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYun -pyGd823IDzANBgkqhkiG9w0BAQUFAAOCAQEAog683+Lt8ONyc3pklL/3cmbYMuRC -dWKuh+vy1dneVrOfzM4UKLkNl2BcEkxY5NM9g0lFWJc1aRqoR+pWxnmrEthngYTf -fwk8lOa4JiwgvT2zKIn3X/8i4peEH+ll74fg38FnSbNd67IJKusm7Xi+fT8r87cm -NW1fiQG2SVufAQWbqz0lwcy2f8Lxb4bG+mRo64EtlOtCt/qMHt1i8b5QZ7dsvfPx -H2sMNgcWfzd8qVttevESRmCD1ycEvkvOl77DZypoEd+A5wwzZr8TDRRu838fYxAe -+o0bJW1sj6W3YQGx0qMmoRBxna3iw/nDmVG3KwcIzi7mULKn+gpFL6Lw8g== ------END CERTIFICATE----- - diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index 9cb887e2dc74..d8087a4db387 100755 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,7 +23,6 @@ https://github.com/bitcoin/bitcoin/blob/37a7fe9e440b83e2364d5498931253937abe9294/contrib/zmq/zmq_sub.py """ -import binascii import asyncio import zmq import zmq.asyncio @@ -58,18 +57,18 @@ async def handle(self) : sequence = str(struct.unpack(' $@ - $(AT)touch $@ + touch $@ define check_or_remove_cached @@ -266,7 +279,7 @@ clean-all: clean @rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* powerpc* riscv32* riscv64* s390x* clean: - @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) + @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) *.log install: check-packages $(host_prefix)/share/config.site @@ -285,3 +298,4 @@ $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) .PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources .PHONY: FORCE +$(V).SILENT: diff --git a/depends/README.md b/depends/README.md index 4f3b6df4871a..a8831eb0fce5 100644 --- a/depends/README.md +++ b/depends/README.md @@ -28,7 +28,8 @@ Common `host-platform-triplet`s for cross compilation are: - `i686-pc-linux-gnu` for Linux 32 bit - `x86_64-pc-linux-gnu` for x86 Linux - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin18` for macOS +- `x86_64-apple-darwin` for macOS +- `arm64-apple-darwin` for ARM macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit - `powerpc64-linux-gnu` for Linux POWER 64-bit (big endian) @@ -38,16 +39,15 @@ Common `host-platform-triplet`s for cross compilation are: - `s390x-linux-gnu` for Linux S390X - `armv7a-linux-android` for Android ARM 32 bit - `aarch64-linux-android` for Android ARM 64 bit -- `i686-linux-android` for Android x86 32 bit - `x86_64-linux-android` for Android x86 64 bit -The paths are automatically configured and no other options are needed unless targeting [Android](#Android). +The paths are automatically configured and no other options are needed unless targeting [Android](../doc/build-android.md). ### Install the required dependencies: Ubuntu & Debian #### For macOS cross compilation - sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libz-dev python3-setuptools libtinfo5 xorriso + sudo apt-get install curl bsdmainutils cmake libz-dev python3-setuptools libtinfo5 xorriso Note: You must obtain the macOS SDK before proceeding with a cross-compile. Under the depends directory, create a subdirectory named `SDKs`. @@ -62,7 +62,7 @@ For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-ex Common linux dependencies: - sudo apt-get install make automake cmake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3 patch bison + sudo apt-get install make automake cmake curl g++-multilib libtool binutils bsdmainutils pkg-config python3 patch bison For linux ARM cross compilation: @@ -80,20 +80,13 @@ For linux RISC-V 64-bit cross compilation (there are no packages for 32-bit): sudo apt-get install g++-riscv64-linux-gnu binutils-riscv64-linux-gnu -RISC-V known issue: gcc-7.3.0 and gcc-7.3.1 result in a broken `test_bitcoin` executable (see https://github.com/bitcoin/bitcoin/pull/13543), -this is apparently fixed in gcc-8.1.0. - For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu -### Install the required dependencies: M1-based macOS - -To be able to build the `qt` package, ensure that Rosetta 2 is installed: +### Install the required dependencies: OpenBSD -``` -softwareupdate --install-rosetta -``` + pkg_add bash gtar ### Dependency Options @@ -103,6 +96,8 @@ The following can be set when running make: `make FOO=bar` - `BASE_CACHE`: Built packages will be placed here - `SDK_PATH`: Path where SDKs can be found (used by macOS) - `FALLBACK_DOWNLOAD_PATH`: If a source file can't be fetched, try here before giving up +- `C_STANDARD`: Set the C standard version used. Defaults to `c11`. +- `CXX_STANDARD`: Set the C++ standard version used. Defaults to `c++17`. - `NO_QT`: Don't download/build/cache Qt and its dependencies - `NO_QR`: Don't download/build/cache packages needed for enabling qrencode - `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ @@ -120,7 +115,11 @@ The following can be set when running make: `make FOO=bar` - `BUILD_ID_SALT`: Optional salt to use when generating build package ids - `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the system's `$PATH` rather than the default prebuilt release of Clang - from llvm.org. Clang 8 or later is required. + from llvm.org. Clang 8 or later is required +- `LOG`: Use file-based logging for individual packages. During a package build its log file + resides in the `depends` directory, and the log file is printed out automatically in case + of build error. After successful build log files are moved along with package archives +- `LTO`: Use LTO when building packages. If some packages are not built, for example `make NO_WALLET=1`, the appropriate options will be passed to bitcoin's configure. In this case, `--disable-wallet`. @@ -133,18 +132,6 @@ options will be passed to bitcoin's configure. In this case, `--disable-wallet`. download-linux: run 'make download-linux' to fetch all sources needed for linux builds -### Android - -Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). -In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. - -API levels from 24 to 29 have been tested to work. - -If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. -This is an example command for a default build with no disabled dependencies: - - ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin - ### Other documentation - [description.md](description.md): General description of the depends system diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 001c92842425..8ed82b276df9 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -6,6 +6,7 @@ build_darwin_STRIP:=$(shell xcrun -f strip) build_darwin_OTOOL:=$(shell xcrun -f otool) build_darwin_NM:=$(shell xcrun -f nm) build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +build_darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) build_darwin_SHA256SUM=shasum -a 256 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o @@ -19,5 +20,11 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool) darwin_OTOOL:=$(shell xcrun -f otool) darwin_NM:=$(shell xcrun -f nm) darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) darwin_native_binutils= darwin_native_toolchain= + +x86_64_darwin_CFLAGS += -arch x86_64 +x86_64_darwin_CXXFLAGS += -arch x86_64 +aarch64_darwin_CFLAGS += -arch arm64 +aarch64_darwin_CXXFLAGS += -arch arm64 diff --git a/depends/builders/default.mk b/depends/builders/default.mk index f097db65d603..cc6dec66c2ba 100644 --- a/depends/builders/default.mk +++ b/depends/builders/default.mk @@ -1,18 +1,17 @@ default_build_CC = gcc default_build_CXX = g++ default_build_AR = ar +default_build_TAR = tar default_build_RANLIB = ranlib default_build_STRIP = strip default_build_NM = nm -default_build_OTOOL = otool -default_build_INSTALL_NAME_TOOL = install_name_tool define add_build_tool_func build_$(build_os)_$1 ?= $$(default_build_$1) build_$(build_arch)_$(build_os)_$1 ?= $$(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) endef -$(foreach var,CC CXX AR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL,$(eval $(call add_build_tool_func,$(var)))) +$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL DSYMUTIL,$(eval $(call add_build_tool_func,$(var)))) define add_build_flags_func build_$(build_arch)_$(build_os)_$1 += $(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) diff --git a/depends/builders/freebsd.mk b/depends/builders/freebsd.mk new file mode 100644 index 000000000000..465f58e04dc7 --- /dev/null +++ b/depends/builders/freebsd.mk @@ -0,0 +1,5 @@ +build_freebsd_CC=clang +build_freebsd_CXX=clang++ + +build_freebsd_SHA256SUM = shasum -a 256 +build_freebsd_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o diff --git a/depends/builders/netbsd.mk b/depends/builders/netbsd.mk new file mode 100644 index 000000000000..b7cf1f751418 --- /dev/null +++ b/depends/builders/netbsd.mk @@ -0,0 +1,2 @@ +build_netbsd_SHA256SUM = shasum -a 256 +build_netbsd_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o diff --git a/depends/builders/openbsd.mk b/depends/builders/openbsd.mk new file mode 100644 index 000000000000..44825d106acf --- /dev/null +++ b/depends/builders/openbsd.mk @@ -0,0 +1,7 @@ +build_openbsd_CC = clang +build_openbsd_CXX = clang++ + +build_openbsd_SHA256SUM = sha256 +build_openbsd_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o + +build_openbsd_TAR = gtar diff --git a/depends/config.site.in b/depends/config.site.in index 5cf107f19b7c..8f6849214d16 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -62,7 +62,7 @@ if test -z "$with_gui" && test -n "@no_qt@"; then with_gui=no fi -if test -n "@debug@" && test -z "@no_qt@" && test "x$with_gui" != xno; then +if test -n "@debug@" && test -z "@no_qt@" && test "$with_gui" != "no"; then with_gui=qt5_debug fi @@ -70,20 +70,23 @@ if test -z "$enable_zmq" && test -n "@no_zmq@"; then enable_zmq=no fi +if test -z "$enable_usdt" && test -n "@no_usdt@"; then + enable_usdt=no +fi + if test "@host_os@" = darwin; then BREW=no - PORT=no fi -PATH="${depends_prefix}/native/bin:${PATH}" +if test -z "$enable_lto" && test -n "@lto@"; then + enable_lto=yes +fi + PKG_CONFIG="$(which pkg-config) --static" -# These two need to remain exported because pkg-config does not see them -# otherwise. That means they must be unexported at the end of configure.ac to -# avoid ruining the cache. Sigh. -export PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig" +PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig" if test -z "@allow_host_packages@"; then - export PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig" + PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig" fi CPPFLAGS="-I${depends_prefix}/include/ ${CPPFLAGS}" @@ -99,7 +102,7 @@ PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PA if test -n "@AR@"; then AR="@AR@" - ac_cv_path_ac_pt_AR="${AR}" + ac_cv_path_AR="${AR}" fi if test -n "@RANLIB@"; then @@ -112,6 +115,28 @@ if test -n "@NM@"; then ac_cv_path_ac_pt_NM="${NM}" fi +if test -n "@STRIP@"; then + STRIP="@STRIP@" + ac_cv_path_ac_pt_STRIP="${STRIP}" +fi + +if test "@host_os@" = darwin; then + if test -n "@OTOOL@"; then + OTOOL="@OTOOL@" + ac_cv_path_OTOOL="${OTOOL}" + fi + + if test -n "@INSTALL_NAME_TOOL@"; then + INSTALL_NAME_TOOL="@INSTALL_NAME_TOOL@" + ac_cv_path_INSTALL_NAME_TOOL="${INSTALL_NAME_TOOL}" + fi + + if test -n "@DSYMUTIL@"; then + DSYMUTIL="@DSYMUTIL@" + ac_cv_path_DSYMUTIL="${DSYMUTIL}" + fi +fi + if test -n "@debug@"; then enable_reduce_exports=no fi diff --git a/depends/funcs.mk b/depends/funcs.mk index 34a030fab7d1..2b21d053b1fa 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -67,6 +67,7 @@ $(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_bui $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz +$(1)_build_log:=$(BASEDIR)/$(1)-$($(1)_version)-$($(1)_build_id).log $(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps @@ -84,11 +85,11 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) #default commands # The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) -$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --no-same-owner --strip-components=1 -xf $$($(1)_source) -$(1)_preprocess_cmds ?= -$(1)_build_cmds ?= -$(1)_config_cmds ?= -$(1)_stage_cmds ?= +$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_TAR) --no-same-owner --strip-components=1 -xf $$($(1)_source) +$(1)_preprocess_cmds ?= true +$(1)_build_cmds ?= true +$(1)_config_cmds ?= true +$(1)_stage_cmds ?= true $(1)_set_vars ?= @@ -136,6 +137,7 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$( $(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig +$(1)_config_env+=PKG_CONFIG_SYSROOT_DIR=/ $(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH) @@ -174,7 +176,7 @@ $(1)_cmake=env CC="$$($(1)_cc)" \ CXX="$$($(1)_cxx)" \ CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ LDFLAGS="$$($(1)_ldflags)" \ - cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" + cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" $$($(1)_cmake_opts) ifeq ($($(1)_type),build) $(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib" else @@ -187,54 +189,59 @@ endif endef define int_add_cmds +ifneq ($(LOG),) +$(1)_logging = >>$$($(1)_build_log) 2>&1 || { if test -f $$($(1)_build_log); then cat $$($(1)_build_log); fi; exit 1; } +endif + $($(1)_fetched): - $(AT)mkdir -p $$(@D) $(SOURCES_PATH) - $(AT)rm -f $$@ - $(AT)touch $$@ - $(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1)) - $(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) - $(AT)touch $$@ + mkdir -p $$(@D) $(SOURCES_PATH) + rm -f $$@ + touch $$@ + cd $$(@D); $($(1)_fetch_cmds) + cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) + touch $$@ $($(1)_extracted): | $($(1)_fetched) - $(AT)echo Extracting $(1)... - $(AT)mkdir -p $$(@D) - $(AT)cd $$(@D); $(call $(1)_extract_cmds,$(1)) - $(AT)touch $$@ + echo Extracting $(1)... + mkdir -p $$(@D) + cd $$(@D); $($(1)_extract_cmds) + touch $$@ $($(1)_preprocessed): | $($(1)_extracted) - $(AT)echo Preprocessing $(1)... - $(AT)mkdir -p $$(@D) $($(1)_patch_dir) - $(AT)$(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) - $(AT)cd $$(@D); $(call $(1)_preprocess_cmds, $(1)) - $(AT)touch $$@ + echo Preprocessing $(1)... + mkdir -p $$(@D) $($(1)_patch_dir) + $(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) + { cd $$(@D); $($(1)_preprocess_cmds); } $$($(1)_logging) + touch $$@ $($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) - $(AT)echo Configuring $(1)... - $(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar --no-same-owner -xf $($(package)_cached); ) - $(AT)mkdir -p $$(@D) - $(AT)+cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1)) - $(AT)touch $$@ + echo Configuring $(1)... + rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), $(build_TAR) --no-same-owner -xf $($(package)_cached); ) + mkdir -p $$(@D) + +{ cd $$(@D); export $($(1)_config_env); $($(1)_config_cmds); } $$($(1)_logging) + touch $$@ $($(1)_built): | $($(1)_configured) - $(AT)echo Building $(1)... - $(AT)mkdir -p $$(@D) - $(AT)+cd $$(@D); $($(1)_build_env) $(call $(1)_build_cmds, $(1)) - $(AT)touch $$@ + echo Building $(1)... + mkdir -p $$(@D) + +{ cd $$(@D); export $($(1)_build_env); $($(1)_build_cmds); } $$($(1)_logging) + touch $$@ $($(1)_staged): | $($(1)_built) - $(AT)echo Staging $(1)... - $(AT)mkdir -p $($(1)_staging_dir)/$(host_prefix) - $(AT)cd $($(1)_build_dir); $($(1)_stage_env) $(call $(1)_stage_cmds, $(1)) - $(AT)rm -rf $($(1)_extract_dir) - $(AT)touch $$@ + echo Staging $(1)... + mkdir -p $($(1)_staging_dir)/$(host_prefix) + +{ cd $($(1)_build_dir); export $($(1)_stage_env); $($(1)_stage_cmds); } $$($(1)_logging) + rm -rf $($(1)_extract_dir) + touch $$@ $($(1)_postprocessed): | $($(1)_staged) - $(AT)echo Postprocessing $(1)... - $(AT)cd $($(1)_staging_prefix_dir); $(call $(1)_postprocess_cmds) - $(AT)touch $$@ + echo Postprocessing $(1)... + cd $($(1)_staging_prefix_dir); $($(1)_postprocess_cmds) + touch $$@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) - $(AT)echo Caching $(1)... - $(AT)cd $$($(1)_staging_dir)/$(host_prefix); find . | sort | tar --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - - $(AT)mkdir -p $$(@D) - $(AT)rm -rf $$(@D) && mkdir -p $$(@D) - $(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@) - $(AT)rm -rf $($(1)_staging_dir) + echo Caching $(1)... + cd $$($(1)_staging_dir)/$(host_prefix); find . | sort | $(build_TAR) --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - + mkdir -p $$(@D) + rm -rf $$(@D) && mkdir -p $$(@D) + mv $$($(1)_staging_dir)/$$(@F) $$(@) + rm -rf $($(1)_staging_dir) + if test -f $($(1)_build_log); then mv $($(1)_build_log) $$(@D); fi $($(1)_cached_checksum): $($(1)_cached) - $(AT)cd $$(@D); $(build_SHA256SUM) $$( $$(@) + cd $$(@D); $(build_SHA256SUM) $$( $$(@) .PHONY: $(1) $(1): | $($(1)_cached_checksum) @@ -264,7 +271,8 @@ $(foreach package,$(packages),$(eval $(package)_type=$(host_arch)_$(host_os))) $(foreach package,$(all_packages),$(eval $(call int_vars,$(package)))) #include package files -$(foreach package,$(all_packages),$(eval include packages/$(package).mk)) +$(foreach native_package,$(native_packages),$(eval include packages/$(native_package).mk)) +$(foreach package,$(packages),$(eval include packages/$(package).mk)) #compute a hash of all files that comprise this package's build recipe $(foreach package,$(all_packages),$(eval $(call int_get_build_recipe_hash,$(package)))) diff --git a/depends/gen_id b/depends/gen_id index ac69ca7ee1fa..7caf8d764d13 100755 --- a/depends/gen_id +++ b/depends/gen_id @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Usage: env [ CC=... ] [ CXX=... ] [ AR=... ] [ RANLIB=... ] [ STRIP=... ] \ -# [ DEBUG=... ] ./build-id [ID_SALT]... +# Usage: env [ CC=... ] [ C_STANDARD=...] [ CXX=... ] [CXX_STANDARD=...] \ +# [ AR=... ] [ RANLIB=... ] [ STRIP=... ] [ DEBUG=... ] \ +# [ LTO=... ] ./build-id [ID_SALT]... # # Prints to stdout a SHA256 hash representing the current toolset, used by # depends/Makefile as a build id for caching purposes (detecting when the @@ -39,12 +40,14 @@ bash -c "${CC} -v" bash -c "${CC} -v -E -xc -o /dev/null - < /dev/null" bash -c "${CC} -v -E -xobjective-c -o /dev/null - < /dev/null" + echo "C_STANDARD=${C_STANDARD}" echo "END CC" echo "BEGIN CXX" bash -c "${CXX} -v" bash -c "${CXX} -v -E -xc++ -o /dev/null - < /dev/null" bash -c "${CXX} -v -E -xobjective-c++ -o /dev/null - < /dev/null" + echo "CXX_STANDARD=${CXX_STANDARD}" echo "END CXX" echo "BEGIN AR" @@ -63,6 +66,10 @@ env | grep '^STRIP_' echo "END STRIP" + echo "BEGIN LTO" + echo "LTO=${LTO}" + echo "END LTO" + echo "END ALL" ) | if [ -n "$DEBUG" ] && command -v tee > /dev/null 2>&1; then # When debugging and `tee` is available, output the preimage to stderr diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk index eabd84bbbe10..b53966dcf8a0 100644 --- a/depends/hosts/android.mk +++ b/depends/hosts/android.mk @@ -1,12 +1,20 @@ ifeq ($(HOST),armv7a-linux-android) -android_AR=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ar android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang -android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ranlib else -android_AR=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ar android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang -android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib endif + +android_CFLAGS=-std=$(C_STANDARD) +android_CXXFLAGS=-std=$(CXX_STANDARD) + +ifneq ($(LTO),) +android_CFLAGS += -flto +android_LDFLAGS += -flto +endif + +android_AR=$(ANDROID_TOOLCHAIN_BIN)/llvm-ar +android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/llvm-ranlib + android_cmake_system=Android diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 5a7ae2df9a34..8fcea35d9875 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,7 +1,7 @@ -OSX_MIN_VERSION=10.14 -OSX_SDK_VERSION=10.15.6 -XCODE_VERSION=12.1 -XCODE_BUILD_ID=12A7403 +OSX_MIN_VERSION=10.15 +OSX_SDK_VERSION=11.0 +XCODE_VERSION=12.2 +XCODE_BUILD_ID=12B45b LD64_VERSION=609 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers @@ -17,6 +17,7 @@ darwin_native_toolchain=native_cctools clang_prog=$(build_prefix)/bin/clang clangxx_prog=$(clang_prog)++ +llvm_config_prog=$(build_prefix)/bin/llvm-config clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version) else @@ -34,11 +35,13 @@ darwin_native_toolchain= # Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") +llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config") clang_resource_dir=$(shell clang -print-resource-dir) +llvm_lib_dir=$(shell $(llvm_config_prog) --libdir) endif -cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL +cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL DSYMUTIL # Make-only lowercase function lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) @@ -109,8 +112,14 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include -darwin_CFLAGS=-pipe -darwin_CXXFLAGS=$(darwin_CFLAGS) +darwin_CFLAGS=-pipe -std=$(C_STANDARD) +darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +darwin_CFLAGS += -flto +darwin_CXXFLAGS += -flto +darwin_LDFLAGS += -flto +endif darwin_release_CFLAGS=-O2 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index 258619a9d059..7c76331ab4a4 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -8,9 +8,8 @@ default_host_AR = $(host_toolchain)ar default_host_RANLIB = $(host_toolchain)ranlib default_host_STRIP = $(host_toolchain)strip default_host_LIBTOOL = $(host_toolchain)libtool -default_host_INSTALL_NAME_TOOL = $(host_toolchain)install_name_tool -default_host_OTOOL = $(host_toolchain)otool default_host_NM = $(host_toolchain)nm +default_host_OBJCOPY = $(host_toolchain)objcopy define add_host_tool_func ifneq ($(filter $(origin $1),undefined default),) @@ -35,5 +34,5 @@ host_$1 = $$($(host_arch)_$(host_os)_$1) host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) endef -$(foreach tool,CC CXX AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL,$(eval $(call add_host_tool_func,$(tool)))) +$(foreach tool,CC CXX AR RANLIB STRIP LIBTOOL NM OBJCOPY OTOOL INSTALL_NAME_TOOL DSYMUTIL,$(eval $(call add_host_tool_func,$(tool)))) $(foreach flags,CFLAGS CXXFLAGS CPPFLAGS LDFLAGS, $(eval $(call add_host_flags_func,$(flags)))) diff --git a/depends/hosts/freebsd.mk b/depends/hosts/freebsd.mk new file mode 100644 index 000000000000..5351d0b90095 --- /dev/null +++ b/depends/hosts/freebsd.mk @@ -0,0 +1,37 @@ +freebsd_CFLAGS=-pipe -std=$(C_STANDARD) +freebsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +freebsd_CFLAGS += -flto +freebsd_CXXFLAGS += -flto +freebsd_LDFLAGS += -flto +endif + +freebsd_release_CFLAGS=-O2 +freebsd_release_CXXFLAGS=$(freebsd_release_CFLAGS) + +freebsd_debug_CFLAGS=-O1 +freebsd_debug_CXXFLAGS=$(freebsd_debug_CFLAGS) + +ifeq (86,$(findstring 86,$(build_arch))) +i686_freebsd_CC=clang -m32 +i686_freebsd_CXX=clang++ -m32 +i686_freebsd_AR=ar +i686_freebsd_RANLIB=ranlib +i686_freebsd_NM=nm +i686_freebsd_STRIP=strip + +x86_64_freebsd_CC=clang -m64 +x86_64_freebsd_CXX=clang++ -m64 +x86_64_freebsd_AR=ar +x86_64_freebsd_RANLIB=ranlib +x86_64_freebsd_NM=nm +x86_64_freebsd_STRIP=strip +else +i686_freebsd_CC=$(default_host_CC) -m32 +i686_freebsd_CXX=$(default_host_CXX) -m32 +x86_64_freebsd_CC=$(default_host_CC) -m64 +x86_64_freebsd_CXX=$(default_host_CXX) -m64 +endif + +freebsd_cmake_system=FreeBSD diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index 07da752492c8..635d3d16da98 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -1,5 +1,15 @@ -linux_CFLAGS=-pipe -linux_CXXFLAGS=$(linux_CFLAGS) +linux_CFLAGS=-pipe -std=$(C_STANDARD) +linux_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +linux_CFLAGS += -flto +linux_CXXFLAGS += -flto +linux_LDFLAGS += -flto + +linux_AR = $(host_toolchain)gcc-ar +linux_NM = $(host_toolchain)gcc-nm +linux_RANLIB = $(host_toolchain)gcc-ranlib +endif linux_release_CFLAGS=-O2 linux_release_CXXFLAGS=$(linux_release_CFLAGS) diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index be5fec570c84..fc1cc1afbe5c 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -1,5 +1,19 @@ -mingw32_CFLAGS=-pipe -mingw32_CXXFLAGS=$(mingw32_CFLAGS) +ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-g++-posix"),) +mingw32_CXX := $(host)-g++-posix +endif + +mingw32_CFLAGS=-pipe -std=$(C_STANDARD) +mingw32_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +mingw32_CFLAGS += -flto +mingw32_CXXFLAGS += -flto +mingw32_LDFLAGS += -flto + +mingw32_AR = $(host_toolchain)gcc-ar +mingw32_NM = $(host_toolchain)gcc-nm +mingw32_RANLIB = $(host_toolchain)gcc-ranlib +endif mingw32_release_CFLAGS=-O2 mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS) @@ -9,4 +23,4 @@ mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -mingw_cmake_system=Windows +mingw32_cmake_system=Windows diff --git a/depends/hosts/netbsd.mk b/depends/hosts/netbsd.mk new file mode 100644 index 000000000000..14121dca20f3 --- /dev/null +++ b/depends/hosts/netbsd.mk @@ -0,0 +1,43 @@ +netbsd_CFLAGS=-pipe -std=$(C_STANDARD) +netbsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +netbsd_CFLAGS += -flto +netbsd_CXXFLAGS += -flto +netbsd_LDFLAGS += -flto + +netbsd_AR = $(host_toolchain)gcc-ar +netbsd_NM = $(host_toolchain)gcc-nm +netbsd_RANLIB = $(host_toolchain)gcc-ranlib +endif + +netbsd_CXXFLAGS=$(netbsd_CFLAGS) + +netbsd_release_CFLAGS=-O2 +netbsd_release_CXXFLAGS=$(netbsd_release_CFLAGS) + +netbsd_debug_CFLAGS=-O1 +netbsd_debug_CXXFLAGS=$(netbsd_debug_CFLAGS) + +ifeq (86,$(findstring 86,$(build_arch))) +i686_netbsd_CC=gcc -m32 +i686_netbsd_CXX=g++ -m32 +i686_netbsd_AR=ar +i686_netbsd_RANLIB=ranlib +i686_netbsd_NM=nm +i686_netbsd_STRIP=strip + +x86_64_netbsd_CC=gcc -m64 +x86_64_netbsd_CXX=g++ -m64 +x86_64_netbsd_AR=ar +x86_64_netbsd_RANLIB=ranlib +x86_64_netbsd_NM=nm +x86_64_netbsd_STRIP=strip +else +i686_netbsd_CC=$(default_host_CC) -m32 +i686_netbsd_CXX=$(default_host_CXX) -m32 +x86_64_netbsd_CC=$(default_host_CC) -m64 +x86_64_netbsd_CXX=$(default_host_CXX) -m64 +endif + +netbsd_cmake_system=NetBSD diff --git a/depends/hosts/openbsd.mk b/depends/hosts/openbsd.mk new file mode 100644 index 000000000000..d330e94d2ed2 --- /dev/null +++ b/depends/hosts/openbsd.mk @@ -0,0 +1,37 @@ +openbsd_CFLAGS=-pipe -std=$(C_STANDARD) +openbsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +openbsd_CFLAGS += -flto +openbsd_CXXFLAGS += -flto +openbsd_LDFLAGS += -flto +endif + +openbsd_release_CFLAGS=-O2 +openbsd_release_CXXFLAGS=$(openbsd_release_CFLAGS) + +openbsd_debug_CFLAGS=-O1 +openbsd_debug_CXXFLAGS=$(openbsd_debug_CFLAGS) + +ifeq (86,$(findstring 86,$(build_arch))) +i686_openbsd_CC=clang -m32 +i686_openbsd_CXX=clang++ -m32 +i686_openbsd_AR=ar +i686_openbsd_RANLIB=ranlib +i686_openbsd_NM=nm +i686_openbsd_STRIP=strip + +x86_64_openbsd_CC=clang -m64 +x86_64_openbsd_CXX=clang++ -m64 +x86_64_openbsd_AR=ar +x86_64_openbsd_RANLIB=ranlib +x86_64_openbsd_NM=nm +x86_64_openbsd_STRIP=strip +else +i686_openbsd_CC=$(default_host_CC) -m32 +i686_openbsd_CXX=$(default_host_CXX) -m32 +x86_64_openbsd_CC=$(default_host_CC) -m64 +x86_64_openbsd_CXX=$(default_host_CXX) -m64 +endif + +openbsd_cmake_system=OpenBSD diff --git a/depends/packages.md b/depends/packages.md index 7ed20ea1297d..4158b46d28d8 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -178,8 +178,8 @@ not sufficient to just say `libprimary`. For us, it's much easier to just link a static `libsecondary` into a shared `libprimary`. Especially because in our case, we are linking against a dummy `libprimary` anyway that we'll throw away. We don't care if the end-user has a -static or dynamic `libseconday`, that's not our concern. With a static -`libseconday`, when we need to link `libprimary` into our executable, there's no +static or dynamic `libsecondary`, that's not our concern. With a static +`libsecondary`, when we need to link `libprimary` into our executable, there's no dependency chain to worry about as `libprimary` has all the symbols. ## Build targets: diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index d45ac3d03f15..262587690c03 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -10,9 +10,14 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking $(package)_config_opts_mingw32=--enable-mingw $(package)_config_opts_linux=--with-pic +$(package)_config_opts_freebsd=--with-pic +$(package)_config_opts_netbsd=--with-pic +$(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic -$(package)_cflags+=-Wno-error=implicit-function-declaration -$(package)_cxxflags=-std=c++17 +$(package)_cflags+=-Wno-error=implicit-function-declaration -Wno-error=format-security +$(package)_cppflags_freebsd=-D_XOPEN_SOURCE=600 +$(package)_cppflags_netbsd=-D_XOPEN_SOURCE=600 +$(package)_cppflags_openbsd=-D_XOPEN_SOURCE=600 $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE endef diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index f879d176f507..e3625d97f566 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,46 +1,10 @@ package=boost -$(package)_version=1_71_0 -$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$(subst _,.,$($(package)_version))/source/ -$(package)_file_name=boost_$($(package)_version).tar.bz2 -$(package)_sha256_hash=d73a8da01e8bf8c7eda40b4c84915071a8c8a0df4a6734537ddde4a8580524ee -$(package)_dependencies=native_b2 - -define $(package)_set_vars -$(package)_config_opts_release=variant=release -$(package)_config_opts_debug=variant=debug -$(package)_config_opts=--layout=tagged --build-type=complete --user-config=user-config.jam -$(package)_config_opts+=threading=multi link=static -sNO_COMPRESSION=1 -$(package)_config_opts_linux=target-os=linux threadapi=pthread runtime-link=shared -$(package)_config_opts_darwin=target-os=darwin runtime-link=shared -$(package)_config_opts_mingw32=target-os=windows binary-format=pe threadapi=win32 runtime-link=static -$(package)_config_opts_x86_64=architecture=x86 address-model=64 -$(package)_config_opts_i686=architecture=x86 address-model=32 -$(package)_config_opts_aarch64=address-model=64 -$(package)_config_opts_armv7a=address-model=32 -ifneq (,$(findstring clang,$($(package)_cxx))) -$(package)_toolset_$(host_os)=clang -else -$(package)_toolset_$(host_os)=gcc -endif -$(package)_config_libraries=filesystem,system,test -$(package)_cxxflags=-std=c++17 -fvisibility=hidden -$(package)_cxxflags_linux=-fPIC -$(package)_cxxflags_android=-fPIC -$(package)_cxxflags_x86_64_darwin=-fcf-protection=full -endef - -define $(package)_preprocess_cmds - echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : \"$($(package)_cflags)\" \"$($(package)_cxxflags)\" \"$($(package)_cppflags)\" \"$($(package)_ldflags)\" \"$($(package)_ar)\" \"$(host_STRIP)\" \"$(host_RANLIB)\" \"$(host_WINDRES)\" : ;" > user-config.jam -endef - -define $(package)_config_cmds - ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) --with-toolset=$($(package)_toolset_$(host_os)) --with-bjam=b2 -endef - -define $(package)_build_cmds - b2 -d2 -j2 -d1 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) stage -endef +$(package)_version=1.80.0 +$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ +$(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.bz2 +$(package)_sha256_hash=1e19565d82e43bc59209a168f5ac899d3ba471d55c7610c677d4ccf2c9c500c0 define $(package)_stage_cmds - b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) install + mkdir -p $($(package)_staging_prefix_dir)/include && \ + cp -r boost $($(package)_staging_prefix_dir)/include endef diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index abeb26545f89..f4778c1ecdc3 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -1,12 +1,20 @@ package=capnp $(package)_version=$(native_$(package)_version) $(package)_download_path=$(native_$(package)_download_path) +$(package)_download_file=$(native_$(package)_download_file) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) +define $(package)_set_vars := +$(package)_config_opts := --with-external-capnp +$(package)_config_opts += CAPNP="$$(native_capnp_prefixbin)/capnp" +$(package)_config_opts += CAPNP_CXX="$$(native_capnp_prefixbin)/capnp-c++" +$(package)_config_opts_android := --disable-shared +endef + define $(package)_config_cmds - $($(package)_autoconf) --with-external-capnp + $($(package)_autoconf) endef define $(package)_build_cmds diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index 902fe43be267..bb203d06f844 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -1,13 +1,18 @@ package=expat -$(package)_version=2.2.7 -$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_7/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=cbc9102f4a31a8dafd42d642e9a3aa31e79a0aedaa1f6efd2795ebc83174ec18 +$(package)_version=2.4.8 +$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_$(subst .,_,$($(package)_version))/ +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=f79b8f904b749e3e0d20afeadecf8249c55b2e32d4ebb089ae378df479dcaf25 +# -D_DEFAULT_SOURCE defines __USE_MISC, which exposes additional +# definitions in endian.h, which are required for a working +# endianess check in configure when building with -flto. define $(package)_set_vars $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking + $(package)_config_opts += --without-xmlwf $(package)_config_opts_linux=--with-pic + $(package)_cppflags += -D_DEFAULT_SOURCE endef define $(package)_config_cmds @@ -23,5 +28,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share lib/cmake lib/*.la endef diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 0d5f94f380c5..c8b2fc33d573 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -1,10 +1,10 @@ package=fontconfig -$(package)_version=2.12.1 +$(package)_version=2.12.6 $(package)_download_path=https://www.freedesktop.org/software/fontconfig/release/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=b449a3e10c47e1d1c7a6ec6e2016cca73d3bd68fbbd4f0ae5cc6b573f7d6c7f3 +$(package)_sha256_hash=cf0c30807d08f6a28ab46c61b8dbd55c97d2f292cf88f3a07d3384687f31f017 $(package)_dependencies=freetype expat -$(package)_patches=remove_char_width_usage.patch gperf_header_regen.patch +$(package)_patches=gperf_header_regen.patch define $(package)_set_vars $(package)_config_opts=--disable-docs --disable-static --disable-libxml2 --disable-iconv @@ -12,7 +12,6 @@ define $(package)_set_vars endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/remove_char_width_usage.patch && \ patch -p1 < $($(package)_patch_dir)/gperf_header_regen.patch endef @@ -29,5 +28,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf var lib/*.la endef diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index a1584608e19f..6f5dbe0f0137 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -1,12 +1,12 @@ package=freetype -$(package)_version=2.7.1 +$(package)_version=2.11.0 $(package)_download_path=https://download.savannah.gnu.org/releases/$(package) -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=3a3bb2c4e15ffb433f2032f50a5b5a92558206822e22bfe8cbe339af4aa82f88 +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=8bee39bd3968c4804b70614a0a3ad597299ad0e824bc8aad5ce8aaf48067bde7 define $(package)_set_vars $(package)_config_opts=--without-zlib --without-png --without-harfbuzz --without-bzip2 --disable-static - $(package)_config_opts += --enable-option-checking + $(package)_config_opts += --enable-option-checking --without-brotli $(package)_config_opts_linux=--with-pic endef @@ -23,5 +23,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share/man lib/*.la endef diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index 4c55c2df04b5..b7e032c0b2d1 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -1,8 +1,8 @@ package=libXau -$(package)_version=1.0.8 +$(package)_version=1.0.9 $(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=fdd477320aeb5cdd67272838722d6b7d544887dfe7de46e1e7cc0c27c2bea4f2 +$(package)_sha256_hash=ccf8cbf0dbf676faa2ea0a6d64bcc3b6746064722b606c8c52917ed00dcb73ec $(package)_dependencies=xproto # When updating this package, check the default value of @@ -30,5 +30,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share lib/*.la endef diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index dad317193c02..5bd12522a7d5 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -12,10 +12,17 @@ define $(package)_set_vars $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic + $(package)_config_opts_freebsd=--with-pic + $(package)_config_opts_netbsd=--with-pic + $(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 endef +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux +endef + define $(package)_config_cmds $($(package)_autoconf) endef @@ -29,5 +36,7 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm lib/*.la && \ + rm include/ev*.h && \ + rm include/event2/*_compat.h endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index 3e5cf5f1603a..6da5693b3fca 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -3,10 +3,20 @@ $(package)_version=$(native_$(package)_version) $(package)_download_path=$(native_$(package)_download_path) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) -$(package)_dependencies=native_$(package) boost capnp +$(package)_dependencies=native_$(package) capnp +ifneq ($(host),$(build)) +$(package)_dependencies += native_capnp +endif + +define $(package)_set_vars := +ifneq ($(host),$(build)) +$(package)_cmake_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp" +$(package)_cmake_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++" +endif +endef define $(package)_config_cmds - $($(package)_cmake) + $($(package)_cmake) . endef define $(package)_build_cmds @@ -14,5 +24,5 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install + $(MAKE) DESTDIR=$($(package)_staging_dir) install-lib endef diff --git a/depends/packages/libnatpmp.mk b/depends/packages/libnatpmp.mk index cdcf8c0bf2c1..2eddc76d9cf4 100644 --- a/depends/packages/libnatpmp.mk +++ b/depends/packages/libnatpmp.mk @@ -1,8 +1,8 @@ package=libnatpmp -$(package)_version=4536032ae32268a45c073a4d5e91bbab4534773a +$(package)_version=07004b97cf691774efebe70404cf22201e4d330d $(package)_download_path=https://github.com/miniupnp/libnatpmp/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=543b460aab26acf91e11d15e17d8798f845304199eea2d76c2f444ec749c5383 +$(package)_sha256_hash=9321953ceb39d07c25463e266e50d0ae7b64676bb3a986d932b18881ed94f1fb define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" diff --git a/depends/packages/libxcb.mk b/depends/packages/libxcb.mk index 710f43959c07..036eaf6560a1 100644 --- a/depends/packages/libxcb.mk +++ b/depends/packages/libxcb.mk @@ -1,30 +1,27 @@ package=libxcb -$(package)_version=1.10 +$(package)_version=1.14 $(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=98d9ab05b636dd088603b64229dd1ab2d2cc02ab807892e107d674f9c3f2d5b5 +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=a55ed6db98d43469801262d81dc2572ed124edc3db31059d4e9916eb9f844c34 $(package)_dependencies=xcb_proto libXau +$(package)_patches = remove_pthread_stubs.patch define $(package)_set_vars -$(package)_config_opts=--disable-static --disable-build-docs --without-doxygen --without-launchd +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen --without-launchd $(package)_config_opts += --disable-dependency-tracking --enable-option-checking -# Because we pass -qt-xcb to Qt, it will compile in a set of xcb helper libraries and extensions, -# so we skip building all of the extensions here. -# More info is available from: https://doc.qt.io/qt-5.9/linux-requirements.html +# Disable unneeded extensions. +# More info is available from: https://doc.qt.io/qt-5.15/linux-requirements.html $(package)_config_opts += --disable-composite --disable-damage --disable-dpms $(package)_config_opts += --disable-dri2 --disable-dri3 --disable-glx -$(package)_config_opts += --disable-present --disable-randr --disable-record -$(package)_config_opts += --disable-render --disable-resource --disable-screensaver -$(package)_config_opts += --disable-shape --disable-sync -$(package)_config_opts += --disable-xevie --disable-xfixes --disable-xfree86-dri -$(package)_config_opts += --disable-xinerama --disable-xinput -$(package)_config_opts += --disable-xprint --disable-selinux --disable-xtest -$(package)_config_opts += --disable-xv --disable-xvmc +$(package)_config_opts += --disable-present --disable-record --disable-resource +$(package)_config_opts += --disable-screensaver --disable-xevie --disable-xfree86-dri +$(package)_config_opts += --disable-xinput --disable-xprint --disable-selinux +$(package)_config_opts += --disable-xtest --disable-xv --disable-xvmc endef define $(package)_preprocess_cmds cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux && \ - sed "s/pthread-stubs//" -i configure + patch -p1 -i $($(package)_patch_dir)/remove_pthread_stubs.patch endef define $(package)_config_cmds diff --git a/depends/packages/libxcb_util.mk b/depends/packages/libxcb_util.mk new file mode 100644 index 000000000000..6f1b9cd7c65e --- /dev/null +++ b/depends/packages/libxcb_util.mk @@ -0,0 +1,32 @@ +package=libxcb_util +$(package)_version=0.4.0 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-$($(package)_version).tar.bz2 +$(package)_sha256_hash=46e49469cb3b594af1d33176cd7565def2be3fa8be4371d62271fabb5eae50e9 +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts = --disable-shared --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +$(package)_config_opts += --with-pic +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_image.mk b/depends/packages/libxcb_util_image.mk new file mode 100644 index 000000000000..d12d67e8e888 --- /dev/null +++ b/depends/packages/libxcb_util_image.mk @@ -0,0 +1,31 @@ +package=libxcb_util_image +$(package)_version=0.4.0 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-image-$($(package)_version).tar.bz2 +$(package)_sha256_hash=2db96a37d78831d643538dd1b595d7d712e04bdccf8896a5e18ce0f398ea2ffc +$(package)_dependencies=libxcb libxcb_util + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts+= --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_keysyms.mk b/depends/packages/libxcb_util_keysyms.mk new file mode 100644 index 000000000000..d4f72dedbea7 --- /dev/null +++ b/depends/packages/libxcb_util_keysyms.mk @@ -0,0 +1,31 @@ +package=libxcb_util_keysyms +$(package)_version=0.4.0 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-keysyms-$($(package)_version).tar.bz2 +$(package)_sha256_hash=0ef8490ff1dede52b7de533158547f8b454b241aa3e4dcca369507f66f216dd9 +$(package)_dependencies=libxcb xproto + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_render.mk b/depends/packages/libxcb_util_render.mk new file mode 100644 index 000000000000..28f1fb073c68 --- /dev/null +++ b/depends/packages/libxcb_util_render.mk @@ -0,0 +1,31 @@ +package=libxcb_util_render +$(package)_version=0.3.9 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-renderutil-$($(package)_version).tar.bz2 +$(package)_sha256_hash=c6e97e48fb1286d6394dddb1c1732f00227c70bd1bedb7d1acabefdd340bea5b +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_wm.mk b/depends/packages/libxcb_util_wm.mk new file mode 100644 index 000000000000..3b905ba4ec55 --- /dev/null +++ b/depends/packages/libxcb_util_wm.mk @@ -0,0 +1,31 @@ +package=libxcb_util_wm +$(package)_version=0.4.1 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-wm-$($(package)_version).tar.bz2 +$(package)_sha256_hash=28bf8179640eaa89276d2b0f1ce4285103d136be6c98262b6151aaee1d3c2a3f +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxkbcommon.mk b/depends/packages/libxkbcommon.mk index 8c6c56545f07..bcdcf671f71a 100644 --- a/depends/packages/libxkbcommon.mk +++ b/depends/packages/libxkbcommon.mk @@ -5,9 +5,14 @@ $(package)_file_name=$(package)-$($(package)_version).tar.xz $(package)_sha256_hash=60ddcff932b7fd352752d51a5c4f04f3d0403230a584df9a2e0d5ed87c486c8b $(package)_dependencies=libxcb +# This package explicitly enables -Werror=array-bounds, which causes build failures +# with GCC 12.1+. Work around that by turning errors back into warnings. +# This workaround would be dropped if the package was updated, as that would require +# a different build system (Meson) define $(package)_set_vars $(package)_config_opts = --enable-option-checking --disable-dependency-tracking $(package)_config_opts += --disable-static --disable-docs +$(package)_cflags += -Wno-error=array-bounds endef define $(package)_preprocess_cmds diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index 99f5b0a8dbcf..7ad2529e470b 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -3,17 +3,20 @@ $(package)_version=2.2.2 $(package)_download_path=https://miniupnp.tuxfamily.org/files/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=888fb0976ba61518276fe1eda988589c700a3f2a69d71089260d75562afd3687 -$(package)_patches=dont_leak_info.patch +$(package)_patches=dont_leak_info.patch respect_mingw_cflags.patch +# Next time this package is updated, ensure that _WIN32_WINNT is still properly set. +# See discussion in https://github.com/bitcoin/bitcoin/pull/25964. define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" $(package)_build_opts_darwin=LIBTOOL="$($(package)_libtool)" -$(package)_build_opts_mingw32=-f Makefile.mingw +$(package)_build_opts_mingw32=-f Makefile.mingw CFLAGS="$($(package)_cflags) -D_WIN32_WINNT=0x0601" $(package)_build_env+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" AR="$($(package)_ar)" endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch + patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch && \ + patch -p1 < $($(package)_patch_dir)/respect_mingw_cflags.patch endef define $(package)_build_cmds diff --git a/depends/packages/native_b2.mk b/depends/packages/native_b2.mk deleted file mode 100644 index aaa37cdcfa7a..000000000000 --- a/depends/packages/native_b2.mk +++ /dev/null @@ -1,20 +0,0 @@ -package=native_b2 -$(package)_version=$(boost_version) -$(package)_download_path=$(boost_download_path) -$(package)_file_name=$(boost_file_name) -$(package)_sha256_hash=$(boost_sha256_hash) -$(package)_build_subdir=tools/build/src/engine -ifneq (,$(findstring clang,$($(package)_cxx))) -$(package)_toolset_$(host_os)=clang -else -$(package)_toolset_$(host_os)=gcc -endif - -define $(package)_build_cmds - CXX="$($(package)_cxx)" CXXFLAGS="$($(package)_cxxflags)" ./build.sh "$($(package)_toolset_$(host_os))" -endef - -define $(package)_stage_cmds - mkdir -p "$($(package)_staging_prefix_dir)"/bin/ && \ - cp b2 "$($(package)_staging_prefix_dir)"/bin/ -endef diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 885207fce9af..03e9002ecd75 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -7,15 +7,25 @@ $(package)_build_subdir=cctools $(package)_dependencies=native_libtapi define $(package)_set_vars - $(package)_config_opts=--target=$(host) + $(package)_config_opts=--target=$(host) --enable-lto-support + $(package)_config_opts+=--with-llvm-config=$(llvm_config_prog) $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib - ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) - $(package)_config_opts+=--enable-lto-support --with-llvm-config=$(build_prefix)/bin/llvm-config - endif $(package)_cc=$(clang_prog) $(package)_cxx=$(clangxx_prog) endef +ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +define $(package)_preprocess_cmds + mkdir -p $($(package)_staging_prefix_dir)/lib && \ + cp $(llvm_lib_dir)/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools +endef +else +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools +endef +endif + define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk index 25ac77c1a34a..f2712294ab20 100644 --- a/depends/packages/native_clang.mk +++ b/depends/packages/native_clang.mk @@ -2,11 +2,9 @@ package=native_clang $(package)_version=10.0.1 $(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) ifneq (,$(findstring aarch64,$(BUILD))) -$(package)_download_file=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz $(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz $(package)_sha256_hash=90dc69a4758ca15cd0ffa45d07fbf5bf4309d47d2c7745a9f0735ecffde9c31f else -$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz $(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz $(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231 endif @@ -18,15 +16,13 @@ endef define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin && \ - mkdir -p $($(package)_staging_prefix_dir)/include && \ + mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \ cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \ cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \ + cp include/llvm-c/ExternC.h $($(package)_staging_prefix_dir)/include/llvm-c && \ + cp include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c && \ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -rf lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ -endef - -define $(package)_postprocess_cmds - rmdir include + cp -r lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ endef diff --git a/depends/packages/native_ds_store.mk b/depends/packages/native_ds_store.mk index 44108925a4f3..51a95f48ef7b 100644 --- a/depends/packages/native_ds_store.mk +++ b/depends/packages/native_ds_store.mk @@ -1,6 +1,6 @@ package=native_ds_store $(package)_version=1.3.0 -$(package)_download_path=https://github.com/al45tair/ds_store/archive/ +$(package)_download_path=https://github.com/dmgbuild/ds_store/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=76b3280cd4e19e5179defa23fb594a9dd32643b0c80d774bd3108361d94fb46d $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages diff --git a/depends/packages/native_libdmg-hfsplus.mk b/depends/packages/native_libdmg-hfsplus.mk deleted file mode 100644 index c7c8adef4158..000000000000 --- a/depends/packages/native_libdmg-hfsplus.mk +++ /dev/null @@ -1,24 +0,0 @@ -package=native_libdmg-hfsplus -$(package)_version=7ac55ec64c96f7800d9818ce64c79670e7f02b67 -$(package)_download_path=https://github.com/planetbeing/libdmg-hfsplus/archive -$(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=56fbdc48ec110966342f0ecddd6f8f89202f4143ed2a3336e42bbf88f940850c -$(package)_build_subdir=build -$(package)_patches=remove-libcrypto-dependency.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/remove-libcrypto-dependency.patch && \ - mkdir build -endef - -define $(package)_config_cmds - $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" -DCMAKE_SKIP_RPATH="ON" -DCMAKE_EXE_LINKER_FLAGS="-static" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" .. -endef - -define $(package)_build_cmds - $(MAKE) -C dmg -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C dmg install -endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index 14653ce9fb0b..e647afba5f35 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,12 +1,12 @@ package=native_libmultiprocess -$(package)_version=d576d975debdc9090bd2582f83f49c76c0061698 +$(package)_version=1af83d15239ccfa7e47b8764029320953dd7fdf1 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=9f8b055c8bba755dc32fe799b67c20b91e7b13e67cadafbc54c0f1def057a370 +$(package)_sha256_hash=e5587d3feedc7f8473f178a89b94163a11076629825d664964799bbbd5844da5 $(package)_dependencies=native_capnp define $(package)_config_cmds - $($(package)_cmake) + $($(package)_cmake) . endef define $(package)_build_cmds @@ -14,5 +14,5 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install + $(MAKE) DESTDIR=$($(package)_staging_dir) install-bin endef diff --git a/depends/packages/native_libtapi.mk b/depends/packages/native_libtapi.mk index 60b898da5f2f..052bb2368933 100644 --- a/depends/packages/native_libtapi.mk +++ b/depends/packages/native_libtapi.mk @@ -1,7 +1,6 @@ package=native_libtapi $(package)_version=664b8414f89612f2dfd35a9b679c345aa5389026 $(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive -$(package)_download_file=$($(package)_version).tar.gz $(package)_file_name=$($(package)_version).tar.gz $(package)_sha256_hash=62e419c12d1c9fad67cc1cd523132bc00db050998337c734c15bc8d73cc02b61 @@ -14,7 +13,5 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - ./install.sh && \ - mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp src/llvm/include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c + ./install.sh endef diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index 783f87ca7c04..ddd631186edf 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -1,6 +1,6 @@ package=native_mac_alias $(package)_version=2.2.0 -$(package)_download_path=https://github.com/al45tair/mac_alias/archive/ +$(package)_download_path=https://github.com/dmgbuild/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=421e6d7586d1f155c7db3e7da01ca0dacc9649a509a253ad7077b70174426499 $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 9094e327ef5b..998cc0221c64 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,10 +1,12 @@ packages:=boost libevent -qrencode_packages = qrencode +qrencode_linux_packages = qrencode +qrencode_android_packages = qrencode +qrencode_darwin_packages = qrencode +qrencode_mingw32_packages = qrencode -qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig libxkbcommon +qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm qt_android_packages=qt - qt_darwin_packages=qt qt_mingw32_packages=qt @@ -19,12 +21,12 @@ natpmp_packages=libnatpmp multiprocess_packages = libmultiprocess capnp multiprocess_native_packages = native_libmultiprocess native_capnp -darwin_native_packages = native_ds_store native_mac_alias +usdt_linux_packages=systemtap -$(host_arch)_$(host_os)_native_packages += native_b2 +darwin_native_packages = native_ds_store native_mac_alias ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_libtapi native_libdmg-hfsplus +darwin_native_packages += native_cctools native_libtapi ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) darwin_native_packages+= native_clang diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 0c3ef8c82f1f..2f7ddf6a5f66 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,42 +1,52 @@ -PACKAGE=qt -$(package)_version=5.12.11 -$(package)_download_path=https://download.qt.io/official_releases/qt/5.12/$($(package)_version)/submodules -$(package)_suffix=everywhere-src-$($(package)_version).tar.xz +package=qt +$(package)_version=5.15.5 +$(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules +$(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=1c1b4e33137ca77881074c140d54c3c9747e845a31338cfe8680f171f0bc3a39 -$(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon +$(package)_sha256_hash=0c42c799aa7c89e479a07c451bf5a301e291266ba789e81afc18f95049524edc +$(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert -$(package)_patches = qt.pro qttools_src.pro -$(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch -$(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch -$(package)_patches+= no_sdk_version_check.patch -$(package)_patches+= fix_lib_paths.patch fix_android_pch.patch -$(package)_patches+= qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch +$(package)_patches = qt.pro +$(package)_patches += qttools_src.pro +$(package)_patches += mac-qmake.conf +$(package)_patches += fix_qt_pkgconfig.patch +$(package)_patches += no-xlib.patch +$(package)_patches += dont_hardcode_x86_64.patch +$(package)_patches += fix_montery_include.patch +$(package)_patches += fix_android_jni_static.patch +$(package)_patches += dont_hardcode_pwd.patch +$(package)_patches += qtbase-moc-ignore-gcc-macro.patch +$(package)_patches += use_android_ndk23.patch +$(package)_patches += rcc_hardcode_timestamp.patch +$(package)_patches += duplicate_lcqpafonts.patch +$(package)_patches += fast_fixed_dtoa_no_optimize.patch +$(package)_patches += guix_cross_lib_path.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=577b0668a777eb2b451c61e8d026d79285371597ce9df06b6dee6c814164b7c3 +$(package)_qttranslations_sha256_hash=c92af4171397a0ed272330b4fa0669790fcac8d050b07c8b8cc565ebeba6735e $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=98b2aaca230458f65996f3534fd471d2ffd038dd58ac997c0589c06dc2385b4f +$(package)_qttools_sha256_hash=6d0778b71b2742cb527561791d1d3d255366163d54a10f78c683a398f09ffc6c $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) define $(package)_set_vars +$(package)_config_env = QT_MAC_SDK_NO_VERSION_CHECK=1 $(package)_config_opts_release = -release $(package)_config_opts_release += -silent $(package)_config_opts_debug = -debug $(package)_config_opts_debug += -optimized-tools $(package)_config_opts += -bindir $(build_prefix)/bin -$(package)_config_opts += -c++std c++1z +$(package)_config_opts += -c++std c++17 $(package)_config_opts += -confirm-license $(package)_config_opts += -hostprefix $(build_prefix) $(package)_config_opts += -no-compile-examples $(package)_config_opts += -no-cups $(package)_config_opts += -no-egl $(package)_config_opts += -no-eglfs -$(package)_config_opts += -no-freetype +$(package)_config_opts += -no-evdev $(package)_config_opts += -no-gif $(package)_config_opts += -no-glib $(package)_config_opts += -no-icu @@ -47,10 +57,12 @@ $(package)_config_opts += -no-linuxfb $(package)_config_opts += -no-libjpeg $(package)_config_opts += -no-libproxy $(package)_config_opts += -no-libudev +$(package)_config_opts += -no-mimetype-database $(package)_config_opts += -no-mtdev $(package)_config_opts += -no-openssl $(package)_config_opts += -no-openvg $(package)_config_opts += -no-reduce-relocations +$(package)_config_opts += -no-schannel $(package)_config_opts += -no-sctp $(package)_config_opts += -no-securetransport $(package)_config_opts += -no-sql-db2 @@ -64,6 +76,7 @@ $(package)_config_opts += -no-sql-sqlite $(package)_config_opts += -no-sql-sqlite2 $(package)_config_opts += -no-system-proxies $(package)_config_opts += -no-use-gold-linker +$(package)_config_opts += -no-zstd $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests $(package)_config_opts += -nomake tools @@ -101,6 +114,7 @@ $(package)_config_opts += -no-feature-sqlmodel $(package)_config_opts += -no-feature-statemachine $(package)_config_opts += -no-feature-syntaxhighlighter $(package)_config_opts += -no-feature-textbrowser +$(package)_config_opts += -no-feature-textmarkdownwriter $(package)_config_opts += -no-feature-textodfwriter $(package)_config_opts += -no-feature-topleveldomain $(package)_config_opts += -no-feature-udpsocket @@ -116,8 +130,12 @@ $(package)_config_opts_darwin = -no-dbus $(package)_config_opts_darwin += -no-opengl $(package)_config_opts_darwin += -pch $(package)_config_opts_darwin += -no-feature-corewlan +$(package)_config_opts_darwin += -no-freetype $(package)_config_opts_darwin += QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) +# Optimizing using > -O1 causes non-determinism when building across arches. +$(package)_config_opts_aarch64_darwin += "QMAKE_CFLAGS_OPTIMIZE_FULL = -O1" + ifneq ($(build_os),darwin) $(package)_config_opts_darwin += -xplatform macx-clang-linux $(package)_config_opts_darwin += -device-option MAC_SDK_PATH=$(OSX_SDK) @@ -127,10 +145,12 @@ $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) $(package)_config_opts_darwin += -device-option XCODE_VERSION=$(XCODE_VERSION) endif -# for macOS on Apple Silicon (ARM) see https://bugreports.qt.io/browse/QTBUG-85279 +ifneq ($(build_arch),$(host_arch)) $(package)_config_opts_aarch64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=arm64 +$(package)_config_opts_x86_64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=x86_64 +endif -$(package)_config_opts_linux = -qt-xcb +$(package)_config_opts_linux = -xcb $(package)_config_opts_linux += -no-xcb-xlib $(package)_config_opts_linux += -no-feature-xlib $(package)_config_opts_linux += -system-freetype @@ -138,21 +158,23 @@ $(package)_config_opts_linux += -fontconfig $(package)_config_opts_linux += -no-opengl $(package)_config_opts_linux += -no-feature-vulkan $(package)_config_opts_linux += -dbus-runtime -$(package)_config_opts_arm_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ -$(package)_config_opts_i686_linux = -xplatform linux-g++-32 -$(package)_config_opts_x86_64_linux = -xplatform linux-g++-64 -$(package)_config_opts_aarch64_linux = -xplatform linux-aarch64-gnu-g++ -$(package)_config_opts_powerpc64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ -$(package)_config_opts_powerpc64le_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ -$(package)_config_opts_riscv64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ -$(package)_config_opts_s390x_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ +ifneq ($(LTO),) +$(package)_config_opts_linux += -ltcg +endif +$(package)_config_opts_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ +ifneq (,$(findstring -stdlib=libc++,$($(1)_cxx))) +$(package)_config_opts_x86_64_linux = -xplatform linux-clang-libc++ +endif $(package)_config_opts_mingw32 = -no-opengl $(package)_config_opts_mingw32 += -no-dbus +$(package)_config_opts_mingw32 += -no-freetype $(package)_config_opts_mingw32 += -xplatform win32-g++ $(package)_config_opts_mingw32 += "QMAKE_CFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" -$(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" +$(package)_config_opts_mingw32 += "QMAKE_CXX = '$($(package)_cxx)'" +$(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cxxflags) $($(package)_cppflags)'" $(package)_config_opts_mingw32 += "QMAKE_LFLAGS = '$($(package)_ldflags)'" +$(package)_config_opts_mingw32 += "QMAKE_LIB = '$($(package)_ar) rc'" $(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" $(package)_config_opts_mingw32 += -pch @@ -160,10 +182,7 @@ $(package)_config_opts_android = -xplatform android-clang $(package)_config_opts_android += -android-sdk $(ANDROID_SDK) $(package)_config_opts_android += -android-ndk $(ANDROID_NDK) $(package)_config_opts_android += -android-ndk-platform android-$(ANDROID_API_LEVEL) -$(package)_config_opts_android += -device-option CROSS_COMPILE="$(host)-" $(package)_config_opts_android += -egl -$(package)_config_opts_android += -qpa xcb -$(package)_config_opts_android += -no-eglfs $(package)_config_opts_android += -no-dbus $(package)_config_opts_android += -opengl es2 $(package)_config_opts_android += -qt-freetype @@ -176,7 +195,6 @@ $(package)_config_opts_android += -no-feature-vulkan $(package)_config_opts_aarch64_android += -android-arch arm64-v8a $(package)_config_opts_armv7a_android += -android-arch armeabi-v7a $(package)_config_opts_x86_64_android += -android-arch x86_64 -$(package)_config_opts_i686_android += -android-arch i686 endef define $(package)_fetch_cmds @@ -192,11 +210,11 @@ define $(package)_extract_cmds echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir qtbase && \ - tar --no-same-owner --strip-components=1 -xf $($(package)_source) -C qtbase && \ + $(build_TAR) --no-same-owner --strip-components=1 -xf $($(package)_source) -C qtbase && \ mkdir qttranslations && \ - tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ + $(build_TAR) --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ mkdir qttools && \ - tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools + $(build_TAR) --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools endef # Preprocessing steps work as follows: @@ -206,49 +224,44 @@ endef # 2. Create a macOS-Clang-Linux mkspec using our mac-qmake.conf. # # 3. After making a copy of the mkspec for the linux-arm-gnueabi host, named -# bitcoin-linux-g++, replace instances of linux-arm-gnueabi with $(host). This -# way we can generically support hosts like riscv64-linux-gnu, which Qt doesn't -# ship a mkspec for. See it's usage in config_opts_* above. +# bitcoin-linux-g++, replace tool names with $($($(package)_type)_TOOL). # # 4. Put our C, CXX and LD FLAGS into gcc-base.conf. Only used for non-host builds. # -# 5. Do similar for the win32-g++ mkspec. -# -# 6. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. -# -# 7. Adjust a regex in toolchain.prf, to accommodate Guix's usage of -# CROSS_LIBRARY_PATH. See #15277. +# 5. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. define $(package)_preprocess_cmds cp $($(package)_patch_dir)/qt.pro qt.pro && \ cp $($(package)_patch_dir)/qttools_src.pro qttools/src/src.pro && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_android_qmake_conf.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_android_pch.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ - patch -p1 -i $($(package)_patch_dir)/no_sdk_version_check.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ + patch -p1 -i $($(package)_patch_dir)/dont_hardcode_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_montery_include.patch && \ + patch -p1 -i $($(package)_patch_dir)/use_android_ndk23.patch && \ + patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ + patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ + patch -p1 -i $($(package)_patch_dir)/fast_fixed_dtoa_no_optimize.patch && \ + patch -p1 -i $($(package)_patch_dir)/guix_cross_lib_path.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \ - sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-gcc|$($($(package)_type)_CC)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-g++|$($($(package)_type)_CXX)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-ar|$($($(package)_type)_AR)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-objcopy|$($($(package)_type)_OBJCOPY)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-nm|$($($(package)_type)_NM)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-strip|$($($(package)_type)_STRIP)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ sed -i.old "s|QMAKE_CC = \$$$$\$$$${CROSS_COMPILE}clang|QMAKE_CC = $($(package)_cc)|" qtbase/mkspecs/common/clang.conf && \ - sed -i.old "s|QMAKE_CXX = \$$$$\$$$${CROSS_COMPILE}clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf && \ - sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf + sed -i.old "s|QMAKE_CXX = \$$$$\$$$${CROSS_COMPILE}clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf endef define $(package)_config_cmds - export PKG_CONFIG_SYSROOT_DIR=/ && \ - export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ cd qtbase && \ ./configure -top-level $($(package)_config_opts) endef diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 5b3a61b239c6..820d724214a2 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -1,12 +1,19 @@ package=sqlite -$(package)_version=3320100 -$(package)_download_path=https://sqlite.org/2020/ +$(package)_version=3380500 +$(package)_download_path=https://sqlite.org/2022/ $(package)_file_name=sqlite-autoconf-$($(package)_version).tar.gz -$(package)_sha256_hash=486748abfb16abd8af664e3a5f03b228e5f124682b0c942e157644bf6fff7d10 +$(package)_sha256_hash=5af07de982ba658fd91a03170c945f99c971f6955bc79df3266544373e39869c define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking $(package)_config_opts_linux=--with-pic +$(package)_config_opts_freebsd=--with-pic +$(package)_config_opts_netbsd=--with-pic +$(package)_config_opts_openbsd=--with-pic +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . endef define $(package)_config_cmds diff --git a/depends/packages/systemtap.mk b/depends/packages/systemtap.mk new file mode 100644 index 000000000000..a57f1b6d36a5 --- /dev/null +++ b/depends/packages/systemtap.mk @@ -0,0 +1,12 @@ +package=systemtap +$(package)_version=4.7 +$(package)_download_path=https://sourceware.org/systemtap/ftp/releases/ +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=43a0a3db91aa4d41e28015b39a65e62059551f3cc7377ebf3a3a5ca7339e7b1f +$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch && \ + mkdir -p $($(package)_staging_prefix_dir)/include/sys && \ + cp includes/sys/sdt.h $($(package)_staging_prefix_dir)/include/sys/sdt.h +endef diff --git a/depends/packages/xcb_proto.mk b/depends/packages/xcb_proto.mk index 01203a0718af..9be822506dbc 100644 --- a/depends/packages/xcb_proto.mk +++ b/depends/packages/xcb_proto.mk @@ -1,8 +1,8 @@ package=xcb_proto -$(package)_version=1.10 -$(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=xcb-proto-$($(package)_version).tar.bz2 -$(package)_sha256_hash=7ef40ddd855b750bc597d2a435da21e55e502a0fefa85b274f2c922800baaf05 +$(package)_version=1.14.1 +$(package)_download_path=https://xorg.freedesktop.org/archive/individual/proto +$(package)_file_name=xcb-proto-$($(package)_version).tar.xz +$(package)_sha256_hash=f04add9a972ac334ea11d9d7eb4fc7f8883835da3e4859c9afa971efdf57fcc3 define $(package)_config_cmds $($(package)_autoconf) @@ -17,6 +17,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - find -name "*.pyc" -delete && \ - find -name "*.pyo" -delete + rm -rf lib/python*/site-packages/xcbgen/__pycache__ endef diff --git a/depends/packages/xproto.mk b/depends/packages/xproto.mk index 6bd867d02b20..7a43c52faf43 100644 --- a/depends/packages/xproto.mk +++ b/depends/packages/xproto.mk @@ -1,8 +1,8 @@ package=xproto -$(package)_version=7.0.26 +$(package)_version=7.0.31 $(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f +$(package)_sha256_hash=c6f9747da0bd3a95f86b17fb8dd5e717c8f3ab7f0ece3ba1b247899ec1ef7747 define $(package)_set_vars $(package)_config_opts=--without-fop --without-xmlto --without-xsltproc --disable-specs diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 3b7f3690a4ac..267ed1125318 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -1,26 +1,31 @@ package=zeromq -$(package)_version=4.3.1 +$(package)_version=4.3.4 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb -$(package)_patches=remove_libstd_link.patch +$(package)_sha256_hash=c593001a89f5a85dd2ddf564805deb860e02471171b3f204944857336295c3e5 +$(package)_patches=remove_libstd_link.patch netbsd_kevent_void.patch define $(package)_set_vars - $(package)_config_opts=--without-docs --disable-shared --disable-curve --disable-curve-keygen --disable-perf + $(package)_config_opts = --without-docs --disable-shared --disable-valgrind + $(package)_config_opts += --disable-perf --disable-curve-keygen --disable-curve --disable-libbsd $(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking $(package)_config_opts_linux=--with-pic + $(package)_config_opts_freebsd=--with-pic + $(package)_config_opts_netbsd=--with-pic + $(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic - $(package)_cxxflags=-std=c++17 endef define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ + patch -p1 < $($(package)_patch_dir)/netbsd_kevent_void.patch && \ cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config endef define $(package)_config_cmds + ./autogen.sh && \ $($(package)_autoconf) endef diff --git a/depends/patches/fontconfig/gperf_header_regen.patch b/depends/patches/fontconfig/gperf_header_regen.patch index 3ffd1674e03b..b1a70d5fb12c 100644 --- a/depends/patches/fontconfig/gperf_header_regen.patch +++ b/depends/patches/fontconfig/gperf_header_regen.patch @@ -13,12 +13,12 @@ diff --git a/src/Makefile.in b/src/Makefile.in index f4626ad..4ae1b00 100644 --- a/src/Makefile.in +++ b/src/Makefile.in -@@ -903,7 +903,7 @@ fcobjshash.gperf: fcobjshash.gperf.h fcobjs.h +@@ -912,7 +912,7 @@ ' - > $@.tmp && \ - mv -f $@.tmp $@ || ( $(RM) $@.tmp && false ) + mv -f $@.tmp fcobjshash.gperf && touch $@ || ( $(RM) $@.tmp && false ) --fcobjshash.h: fcobjshash.gperf +-fcobjshash.h: Makefile fcobjshash.gperf +fcobjshash.h: - $(AM_V_GEN) $(GPERF) -m 100 $< > $@.tmp && \ + $(AM_V_GEN) $(GPERF) --pic -m 100 fcobjshash.gperf > $@.tmp && \ mv -f $@.tmp $@ || ( $(RM) $@.tmp && false ) diff --git a/depends/patches/fontconfig/remove_char_width_usage.patch b/depends/patches/fontconfig/remove_char_width_usage.patch deleted file mode 100644 index 9f69081890c3..000000000000 --- a/depends/patches/fontconfig/remove_char_width_usage.patch +++ /dev/null @@ -1,62 +0,0 @@ -commit 28165a9b078583dc8e9e5c344510e37582284cef -Author: fanquake -Date: Mon Aug 17 20:35:42 2020 +0800 - - Remove usage of CHAR_WIDTH - - CHAR_WIDTH which is reserved and clashes with glibc 2.25+ - - See #10851. - -diff --git a/fontconfig/fontconfig.h b/fontconfig/fontconfig.h -index 5c72b22..843c532 100644 ---- a/fontconfig/fontconfig.h -+++ b/fontconfig/fontconfig.h -@@ -128,7 +128,7 @@ typedef int FcBool; - #define FC_USER_CACHE_FILE ".fonts.cache-" FC_CACHE_VERSION - - /* Adjust outline rasterizer */ --#define FC_CHAR_WIDTH "charwidth" /* Int */ -+#define FC_CHARWIDTH "charwidth" /* Int */ - #define FC_CHAR_HEIGHT "charheight"/* Int */ - #define FC_MATRIX "matrix" /* FcMatrix */ - -diff --git a/src/fcobjs.h b/src/fcobjs.h -index 1fc4f65..d27864b 100644 ---- a/src/fcobjs.h -+++ b/src/fcobjs.h -@@ -51,7 +51,7 @@ FC_OBJECT (DPI, FcTypeDouble, NULL) - FC_OBJECT (RGBA, FcTypeInteger, NULL) - FC_OBJECT (SCALE, FcTypeDouble, NULL) - FC_OBJECT (MINSPACE, FcTypeBool, NULL) --FC_OBJECT (CHAR_WIDTH, FcTypeInteger, NULL) -+FC_OBJECT (CHARWIDTH, FcTypeInteger, NULL) - FC_OBJECT (CHAR_HEIGHT, FcTypeInteger, NULL) - FC_OBJECT (MATRIX, FcTypeMatrix, NULL) - FC_OBJECT (CHARSET, FcTypeCharSet, FcCompareCharSet) -diff --git a/src/fcobjshash.gperf b/src/fcobjshash.gperf -index 80a0237..eb4ad84 100644 ---- a/src/fcobjshash.gperf -+++ b/src/fcobjshash.gperf -@@ -44,7 +44,7 @@ int id; - "rgba",FC_RGBA_OBJECT - "scale",FC_SCALE_OBJECT - "minspace",FC_MINSPACE_OBJECT --"charwidth",FC_CHAR_WIDTH_OBJECT -+"charwidth",FC_CHARWIDTH_OBJECT - "charheight",FC_CHAR_HEIGHT_OBJECT - "matrix",FC_MATRIX_OBJECT - "charset",FC_CHARSET_OBJECT -diff --git a/src/fcobjshash.h b/src/fcobjshash.h -index 5a4d1ea..4e66bb0 100644 ---- a/src/fcobjshash.h -+++ b/src/fcobjshash.h -@@ -284,7 +284,7 @@ FcObjectTypeLookup (register const char *str, register unsigned int len) - {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str43,FC_CHARSET_OBJECT}, - {-1}, - #line 47 "fcobjshash.gperf" -- {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str45,FC_CHAR_WIDTH_OBJECT}, -+ {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str45,FC_CHARWIDTH_OBJECT}, - #line 48 "fcobjshash.gperf" - {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str46,FC_CHAR_HEIGHT_OBJECT}, - #line 55 "fcobjshash.gperf" diff --git a/depends/patches/libxcb/remove_pthread_stubs.patch b/depends/patches/libxcb/remove_pthread_stubs.patch new file mode 100644 index 000000000000..1f32dea527e2 --- /dev/null +++ b/depends/patches/libxcb/remove_pthread_stubs.patch @@ -0,0 +1,12 @@ +Remove uneeded pthread-stubs dependency +--- a/configure ++++ b/configure +@@ -19695,7 +19695,7 @@ fi + NEEDED="xau >= 0.99.2" + case $host_os in + linux*) ;; +- *) NEEDED="$NEEDED pthread-stubs" ;; ++ *) NEEDED="$NEEDED" ;; + esac + + pkg_failed=no diff --git a/depends/patches/miniupnpc/respect_mingw_cflags.patch b/depends/patches/miniupnpc/respect_mingw_cflags.patch new file mode 100644 index 000000000000..a44580ddab65 --- /dev/null +++ b/depends/patches/miniupnpc/respect_mingw_cflags.patch @@ -0,0 +1,23 @@ +commit fec515a7ac9991a0ee91068fda046b54b191155e +Author: fanquake +Date: Wed Jul 27 15:52:37 2022 +0100 + + build: respect CFLAGS in makefile.mingw + + Similar to the other Makefile. + + Cherry-pick of https://github.com/miniupnp/miniupnp/pull/619. + +diff --git a/Makefile.mingw b/Makefile.mingw +index 2bff7bd..88430d2 100644 +--- a/Makefile.mingw ++++ b/Makefile.mingw +@@ -19,7 +19,7 @@ else + RM = rm -f + endif + #CFLAGS = -Wall -g -DDEBUG -D_WIN32_WINNT=0X501 +-CFLAGS = -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 ++CFLAGS ?= -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 + LDLIBS = -lws2_32 -liphlpapi + # -lwsock32 + # -liphlpapi is needed for GetBestRoute() and GetIpAddrTable() diff --git a/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch b/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch deleted file mode 100644 index f346c8f2cff8..000000000000 --- a/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 3e5fd3fb56bc9ff03beb535979e33dcf83fe1f70 Mon Sep 17 00:00:00 2001 -From: Cory Fields -Date: Thu, 8 May 2014 12:39:42 -0400 -Subject: [PATCH] dmg: remove libcrypto dependency - ---- - dmg/CMakeLists.txt | 16 ---------------- - 1 file changed, 16 deletions(-) - -diff --git a/dmg/CMakeLists.txt b/dmg/CMakeLists.txt -index eec62d6..3969f64 100644 ---- a/dmg/CMakeLists.txt -+++ b/dmg/CMakeLists.txt -@@ -1,12 +1,5 @@ --INCLUDE(FindOpenSSL) - INCLUDE(FindZLIB) - --FIND_LIBRARY(CRYPTO_LIBRARIES crypto -- PATHS -- /usr/lib -- /usr/local/lib -- ) -- - IF(NOT ZLIB_FOUND) - message(FATAL_ERROR "zlib is required for dmg!") - ENDIF(NOT ZLIB_FOUND) -@@ -18,15 +11,6 @@ link_directories(${PROJECT_BINARY_DIR}/common ${PROJECT_BINARY_DIR}/hfs) - - add_library(dmg adc.c base64.c checksum.c dmgfile.c dmglib.c filevault.c io.c partition.c resources.c udif.c) - --IF(OPENSSL_FOUND) -- add_definitions(-DHAVE_CRYPT) -- include_directories(${OPENSSL_INCLUDE_DIR}) -- target_link_libraries(dmg ${CRYPTO_LIBRARIES}) -- IF(WIN32) -- TARGET_LINK_LIBRARIES(dmg gdi32) -- ENDIF(WIN32) --ENDIF(OPENSSL_FOUND) -- - target_link_libraries(dmg common hfs z) - - add_executable(dmg-bin dmg.c) --- -2.22.0 - diff --git a/depends/patches/qt/dont_hardcode_x86_64.patch b/depends/patches/qt/dont_hardcode_x86_64.patch new file mode 100644 index 000000000000..a66426877ad7 --- /dev/null +++ b/depends/patches/qt/dont_hardcode_x86_64.patch @@ -0,0 +1,119 @@ +macOS: Don't hard-code x86_64 as the architecture when using qmake + +Upstream commit: + - Qt 6.1: 9082cc8e8d5a6441dabe5e7a95bc0cd9085b95fe + +For other Qt branches see +https://codereview.qt-project.org/q/I70db7e4c27f0d3da5d0af33cb491d72c312d3fa8 + + +--- old/qtbase/configure.json ++++ new/qtbase/configure.json +@@ -244,11 +244,18 @@ + + "testTypeDependencies": { + "linkerSupportsFlag": [ "use_bfd_linker", "use_gold_linker", "use_lld_linker" ], +- "verifySpec": [ "shared", "use_bfd_linker", "use_gold_linker", "use_lld_linker", "compiler-flags", "qmakeargs", "commit" ], ++ "verifySpec": [ ++ "shared", ++ "use_bfd_linker", "use_gold_linker", "use_lld_linker", ++ "compiler-flags", "qmakeargs", ++ "simulator_and_device", ++ "thread", ++ "commit" ], + "compile": [ "verifyspec" ], + "detectPkgConfig": [ "cross_compile", "machineTuple" ], + "library": [ "pkg-config", "compiler-flags" ], +- "getPkgConfigVariable": [ "pkg-config" ] ++ "getPkgConfigVariable": [ "pkg-config" ], ++ "architecture" : [ "verifyspec" ] + }, + + "testTypeAliases": { +@@ -762,7 +769,7 @@ + }, + "architecture": { + "label": "Architecture", +- "output": [ "architecture" ] ++ "output": [ "architecture", "commitConfig" ] + }, + "pkg-config": { + "label": "Using pkg-config", +diff --git a/configure.pri b/configure.pri +index 49755f7abfd..8be9b10d7d4 100644 +--- old/qtbase/configure.pri ++++ new/qtbase/configure.pri +@@ -662,6 +662,13 @@ defineTest(qtConfOutput_commitOptions) { + write_file($$QT_BUILD_TREE/mkspecs/qdevice.pri, $${currentConfig}.output.devicePro)|error() + } + ++# Output is written after configuring each Qt module, ++# but some tests within a module might depend on the ++# configuration output of previous tests. ++defineTest(qtConfOutput_commitConfig) { ++ qtConfProcessOutput() ++} ++ + # type (empty or 'host'), option name, default value + defineTest(processQtPath) { + out_var = config.rel_input.$${2} +diff --git a/mkspecs/common/macx.conf b/mkspecs/common/macx.conf +index d16b77acb8e..4ba0a8eaa36 100644 +--- old/qtbase/mkspecs/common/macx.conf ++++ new/qtbase/mkspecs/common/macx.conf +@@ -6,7 +6,6 @@ QMAKE_PLATFORM += macos osx macx + QMAKE_MAC_SDK = macosx + + QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.13 +-QMAKE_APPLE_DEVICE_ARCHS = x86_64 + + # Should be 10.15, but as long as the CI builds with + # older SDKs we have to keep this. +diff --git a/mkspecs/features/mac/default_post.prf b/mkspecs/features/mac/default_post.prf +index 92a9112bca6..d888731ec8d 100644 +--- old/qtbase/mkspecs/features/mac/default_post.prf ++++ new/qtbase/mkspecs/features/mac/default_post.prf +@@ -95,6 +95,11 @@ app_extension_api_only { + QMAKE_LFLAGS += $$QMAKE_CFLAGS_APPLICATION_EXTENSION + } + ++# Non-universal builds do not set QMAKE_APPLE_DEVICE_ARCHS, ++# so we pick it up from what the arch test resolved instead. ++isEmpty(QMAKE_APPLE_DEVICE_ARCHS): \ ++ QMAKE_APPLE_DEVICE_ARCHS = $$QT_ARCH ++ + macx-xcode { + qmake_pkginfo_typeinfo.name = QMAKE_PKGINFO_TYPEINFO + !isEmpty(QMAKE_PKGINFO_TYPEINFO): \ +@@ -150,9 +155,6 @@ macx-xcode { + simulator: VALID_SIMULATOR_ARCHS = $$QMAKE_APPLE_SIMULATOR_ARCHS + VALID_ARCHS = $$VALID_DEVICE_ARCHS $$VALID_SIMULATOR_ARCHS + +- isEmpty(VALID_ARCHS): \ +- error("QMAKE_APPLE_DEVICE_ARCHS or QMAKE_APPLE_SIMULATOR_ARCHS must contain at least one architecture") +- + single_arch: VALID_ARCHS = $$first(VALID_ARCHS) + + ACTIVE_ARCHS = $(filter $(EXPORT_VALID_ARCHS), $(ARCHS)) +diff --git a/mkspecs/features/toolchain.prf b/mkspecs/features/toolchain.prf +index efbe7c1e55b..8add6dc8043 100644 +--- old/qtbase/mkspecs/features/toolchain.prf ++++ new/qtbase/mkspecs/features/toolchain.prf +@@ -182,9 +182,14 @@ isEmpty($${target_prefix}.INCDIRS) { + # UIKit simulator platforms will see the device SDK's sysroot in + # QMAKE_DEFAULT_*DIRS, because they're handled in a single build pass. + darwin { +- # Clang doesn't pick up the architecture from the sysroot, and will +- # default to the host architecture, so we need to manually set it. +- cxx_flags += -arch $$QMAKE_APPLE_DEVICE_ARCHS ++ uikit { ++ # Clang doesn't automatically pick up the architecture, just because ++ # we're passing the iOS sysroot below, and we will end up building the ++ # test for the host architecture, resulting in linker errors when ++ # linking against the iOS libraries. We work around this by passing ++ # the architecture explicitly. ++ cxx_flags += -arch $$first(QMAKE_APPLE_DEVICE_ARCHS) ++ } + + uikit:macx-xcode: \ + cxx_flags += -isysroot $$sdk_path_device.value diff --git a/depends/patches/qt/duplicate_lcqpafonts.patch b/depends/patches/qt/duplicate_lcqpafonts.patch new file mode 100644 index 000000000000..c460b51dcff6 --- /dev/null +++ b/depends/patches/qt/duplicate_lcqpafonts.patch @@ -0,0 +1,104 @@ +QtGui: Fix duplication of logging category lcQpaFonts + +Move it to qplatformfontdatabase.h. + +Upstream commit: + - Qt 6.0: ab01885e48873fb2ad71841a3f1627fe4d9cd835 + +--- a/qtbase/src/gui/text/qplatformfontdatabase.cpp ++++ b/qtbase/src/gui/text/qplatformfontdatabase.cpp +@@ -52,6 +52,8 @@ + + QT_BEGIN_NAMESPACE + ++Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") ++ + void qt_registerFont(const QString &familyname, const QString &stylename, + const QString &foundryname, int weight, + QFont::Style style, int stretch, bool antialiased, + +--- a/qtbase/src/gui/text/qplatformfontdatabase.h ++++ b/qtbase/src/gui/text/qplatformfontdatabase.h +@@ -50,6 +50,7 @@ + // + + #include ++#include + #include + #include + #include +@@ -62,6 +63,7 @@ + + QT_BEGIN_NAMESPACE + ++Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) + + class QWritingSystemsPrivate; + + +--- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm ++++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm +@@ -86,8 +86,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") +- + static float SYNTHETIC_ITALIC_SKEW = std::tan(14.f * std::acos(0.f) / 90.f); + + bool QCoreTextFontEngine::ct_getSfntTable(void *user_data, uint tag, uchar *buffer, uint *length) + +--- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h ++++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h +@@ -64,8 +64,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) +- + class QCoreTextFontEngine : public QFontEngine + { + Q_GADGET + +--- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp ++++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp +@@ -68,8 +68,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") +- + #ifndef QT_NO_DIRECTWRITE + // ### fixme: Consider direct linking of dwrite.dll once Windows Vista pre SP2 is dropped (QTBUG-49711) + + +--- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h ++++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h +@@ -63,8 +63,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) +- + class QWindowsFontEngineData + { + Q_DISABLE_COPY_MOVE(QWindowsFontEngineData) + +--- a/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp ++++ b/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp +@@ -40,6 +40,7 @@ + #include "qgenericunixthemes_p.h" + + #include "qpa/qplatformtheme_p.h" ++#include "qpa/qplatformfontdatabase.h" + + #include + #include +@@ -76,7 +77,6 @@ + QT_BEGIN_NAMESPACE + + Q_DECLARE_LOGGING_CATEGORY(qLcTray) +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") + + ResourceHelper::ResourceHelper() + { diff --git a/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch b/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch new file mode 100644 index 000000000000..d4d6539f56dc --- /dev/null +++ b/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch @@ -0,0 +1,20 @@ +Modify the optimisation flags for FastFixedDtoa. +This fixes a non-determinism issue in the asm produced for +this function when cross-compiling on x86_64 and aarch64 for +the arm-linux-gnueabihf HOST. + +--- a/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h ++++ b/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h +@@ -48,9 +48,12 @@ namespace double_conversion { + // + // This method only works for some parameters. If it can't handle the input it + // returns false. The output is null-terminated when the function succeeds. ++#pragma GCC push_options ++#pragma GCC optimize ("-O1") + bool FastFixedDtoa(double v, int fractional_count, + Vector buffer, int* length, int* decimal_point); + ++#pragma GCC pop_options + } // namespace double_conversion + + #endif // DOUBLE_CONVERSION_FIXED_DTOA_H_ diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index a186aeb8f6cf..936b82e1522e 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -1,6 +1,6 @@ --- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp +++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -898,6 +898,14 @@ +@@ -943,6 +943,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); return -1; } diff --git a/depends/patches/qt/fix_android_pch.patch b/depends/patches/qt/fix_android_pch.patch deleted file mode 100644 index bed6e4bb6359..000000000000 --- a/depends/patches/qt/fix_android_pch.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- old/qtbase/mkspecs/common/android-base-head.conf -+++ new/qtbase/mkspecs/common/android-base-head.conf -@@ -73,6 +73,6 @@ CROSS_COMPILE = $$NDK_TOOLCHAIN_PATH/bin/$$NDK_TOOLS_PREFIX- - QMAKE_PCH_OUTPUT_EXT = .gch - - QMAKE_CFLAGS_PRECOMPILE = -x c-header -c ${QMAKE_PCH_INPUT} -o ${QMAKE_PCH_OUTPUT} --QMAKE_CFLAGS_USE_PRECOMPILE = -include ${QMAKE_PCH_OUTPUT_BASE} -+QMAKE_CFLAGS_USE_PRECOMPILE = -include-pch ${QMAKE_PCH_OUTPUT} - QMAKE_CXXFLAGS_PRECOMPILE = -x c++-header -c ${QMAKE_PCH_INPUT} -o ${QMAKE_PCH_OUTPUT} - QMAKE_CXXFLAGS_USE_PRECOMPILE = $$QMAKE_CFLAGS_USE_PRECOMPILE diff --git a/depends/patches/qt/fix_android_qmake_conf.patch b/depends/patches/qt/fix_android_qmake_conf.patch deleted file mode 100644 index 3a8753fd1d74..000000000000 --- a/depends/patches/qt/fix_android_qmake_conf.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- old/qtbase/mkspecs/android-clang/qmake.conf -+++ new/qtbase/mkspecs/android-clang/qmake.conf -@@ -47,7 +47,7 @@ ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so - ANDROID_USE_LLVM = true - - exists($$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so): \ -- ANDROID_CXX_STL_LIBS = -lc++ -+ ANDROID_CXX_STL_LIBS = -lc++_shared - else: \ - ANDROID_CXX_STL_LIBS = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so.$$replace(ANDROID_PLATFORM, "android-", "") diff --git a/depends/patches/qt/fix_lib_paths.patch b/depends/patches/qt/fix_lib_paths.patch deleted file mode 100644 index d1a15373f44d..000000000000 --- a/depends/patches/qt/fix_lib_paths.patch +++ /dev/null @@ -1,193 +0,0 @@ ---- old/qtbase/mkspecs/common/mac.conf -+++ new/qtbase/mkspecs/common/mac.conf -@@ -14,7 +14,6 @@ - - QMAKE_RESOURCE = /Developer/Tools/Rez - QMAKE_EXTENSION_SHLIB = dylib --QMAKE_EXTENSIONS_AUX_SHLIB = tbd - QMAKE_LIBDIR = - - # sdk.prf will prefix the proper SDK sysroot - ---- old/qtbase/mkspecs/features/qmake_use.prf -+++ new/qtbase/mkspecs/features/qmake_use.prf -@@ -22,6 +22,8 @@ - !defined(QMAKE_LIBS_$$nu, var): \ - error("Library '$$lower($$replace(nu, _, -))' is not defined.") - -+ QMAKE_LIBDIR += $$eval(QMAKE_LIBDIR_$$nu) -+ - debug: \ - LIBS$${suffix} += $$eval(QMAKE_LIBS_$${nu}_DEBUG) $$eval(QMAKE_LIBS_$$nu) - else: \ - ---- old/qtbase/mkspecs/features/qt_configure.prf -+++ new/qtbase/mkspecs/features/qt_configure.prf -@@ -526,98 +526,23 @@ - return($$sysrootified) - } - --# libs-var, libs, in-paths, out-paths-var -+# libs-var, libs, in-paths - defineTest(qtConfResolveLibs) { -- ret = true -- paths = $$3 -- out = -- copy = false -- for (l, 2) { -- $$copy { -- copy = false -- out += $$l -- } else: equals(l, "-s") { -- # em++ flag to link libraries from emscripten-ports; passed on literally. -- copy = true -- out += $$l -- } else: contains(l, "^-L.*") { -- lp = $$replace(l, "^-L", ) -- gcc: lp = $$qtGccSysrootifiedPath($$lp) -- !exists($$lp/.) { -- qtLog("Library path $$val_escape(lp) is invalid.") -- ret = false -- } else { -- paths += $$lp -- } -- } else: contains(l, "^-l.*") { -- lib = $$replace(l, "^-l", ) -- lcan = -- integrity:contains(lib, "^.*\\.a") { -- # INTEGRITY compiler searches for exact filename -- # if -l argument has .a suffix -- lcan += $${lib} -- } else: contains(lib, "^:.*") { -- # Use exact filename when -l:filename syntax is used. -- lib ~= s/^:// -- lcan += $${lib} -- } else: unix { -- # Under UNIX, we look for actual shared libraries, in addition -- # to static ones. -- shexts = $$QMAKE_EXTENSION_SHLIB $$QMAKE_EXTENSIONS_AUX_SHLIB -- for (ext, shexts) { -- lcan += $${QMAKE_PREFIX_SHLIB}$${lib}.$${ext} -- } -- lcan += \ -- $${QMAKE_PREFIX_STATICLIB}$${lib}.$${QMAKE_EXTENSION_STATICLIB} -- } else { -- # Under Windows, we look only for static libraries, as even for DLLs -- # one actually links against a static import library. -- mingw { -- lcan += \ -- # MinGW supports UNIX-style library naming in addition to -- # the MSVC style. -- lib$${lib}.dll.a lib$${lib}.a \ -- # Fun fact: prefix-less libraries are also supported. -- $${lib}.dll.a $${lib}.a -- } -- lcan += $${lib}.lib -- } -- l = $$qtConfFindInPathList($$lcan, $$paths $$EXTRA_LIBDIR $$QMAKE_DEFAULT_LIBDIRS) -- isEmpty(l) { -- qtLog("None of [$$val_escape(lcan)] found in [$$val_escape(paths)] and global paths.") -- ret = false -- } else { -- out += $$l -- } -- } else { -- out += $$l -- } -- } -- $$1 = $$out -+ for (path, 3): \ -+ pre_lflags += -L$$path -+ $$1 = $$pre_lflags $$2 - export($$1) -- !isEmpty(4) { -- $$4 = $$paths -- export($$4) -- } -- return($$ret) --} -- --# source-var --defineTest(qtConfResolveAllLibs) { -- ret = true -- !qtConfResolveLibs($${1}.libs, $$eval($${1}.libs), , $${1}.libdirs): \ -- ret = false -- for (b, $${1}.builds._KEYS_): \ -- !qtConfResolveLibs($${1}.builds.$${b}, $$eval($${1}.builds.$${b}), $$eval($${1}.libdirs), ): \ -- ret = false -- return($$ret) -+ return(true) - } - - # libs-var, in-paths, libs - defineTest(qtConfResolvePathLibs) { - ret = true -- gcc: 2 = $$qtGccSysrootifiedPaths($$2) -- for (libdir, 2) { -+ gcc: \ -+ local_paths = $$qtGccSysrootifiedPaths($$2) -+ else: \ -+ local_paths = $$2 -+ for (libdir, local_paths) { - !exists($$libdir/.) { - qtLog("Library path $$val_escape(libdir) is invalid.") - ret = false -@@ -667,8 +592,11 @@ - # includes-var, in-paths, test-object-var - defineTest(qtConfResolvePathIncs) { - ret = true -- gcc: 2 = $$qtGccSysrootifiedPaths($$2) -- for (incdir, 2) { -+ gcc: \ -+ local_paths = $$qtGccSysrootifiedPaths($$2) -+ else: \ -+ local_paths = $$2 -+ for (incdir, local_paths) { - !exists($$incdir/.) { - qtLog("Include path $$val_escape(incdir) is invalid.") - ret = false -@@ -727,6 +655,7 @@ - vars += $$eval(config.commandline.rev_assignments.$${iv}) - defined(config.input.$${iv}, var) { - eval($${1}.builds.$${b} = $$eval(config.input.$${iv})) -+ export($${1}.builds.$${b}) - $${1}.builds._KEYS_ *= $${b} - any = true - } else { -@@ -741,11 +670,14 @@ - export($${1}.builds._KEYS_) - # we also reset the generic libs, to avoid surprises. - $${1}.libs = -+ export($${1}.libs) - } - - # direct libs. overwrites inline libs. -- defined(config.input.$${input}.libs, var): \ -+ defined(config.input.$${input}.libs, var) { - eval($${1}.libs = $$eval(config.input.$${input}.libs)) -+ export($${1}.libs) -+ } - - includes = $$eval(config.input.$${input}.incdir) - -@@ -754,6 +686,7 @@ - !isEmpty(prefix) { - includes += $$prefix/include - $${1}.libs = -L$$prefix/lib $$eval($${1}.libs) -+ export($${1}.libs) - } - - libdir = $$eval(config.input.$${input}.libdir) -@@ -762,11 +695,9 @@ - for (ld, libdir): \ - libs += -L$$ld - $${1}.libs = $$libs $$eval($${1}.libs) -+ export($${1}.libs) - } - -- !qtConfResolveAllLibs($$1): \ -- return(false) -- - !qtConfResolvePathIncs($${1}.includedir, $$includes, $$2): \ - return(false) - diff --git a/depends/patches/qt/fix_limits_header.patch b/depends/patches/qt/fix_limits_header.patch deleted file mode 100644 index e4313770e5a8..000000000000 --- a/depends/patches/qt/fix_limits_header.patch +++ /dev/null @@ -1,44 +0,0 @@ -Fix compiling with GCC 11 - -See: https://bugreports.qt.io/browse/QTBUG-90395. - -Upstream commits: - - Qt 5.15 -- unavailable as open source - - Qt 6.0: b2af6332ea37e45ab230a7a5d2d278f86d961b83 - - Qt 6.1: 9c56d4da2ff631a8c1c30475bd792f6c86bda53c - ---- old/qtbase/src/corelib/global/qendian.h -+++ new/qtbase/src/corelib/global/qendian.h -@@ -44,6 +44,8 @@ - #include - #include - -+#include -+ - // include stdlib.h and hope that it defines __GLIBC__ for glibc-based systems - #include - #include - ---- old/qtbase/src/corelib/tools/qbytearraymatcher.h -+++ new/qtbase/src/corelib/tools/qbytearraymatcher.h -@@ -42,6 +42,8 @@ - - #include - -+#include -+ - QT_BEGIN_NAMESPACE - - - ---- old/qtbase/src/tools/moc/generator.cpp -+++ new/qtbase/src/tools/moc/generator.cpp -@@ -40,6 +40,8 @@ - #include - #include - -+#include -+ - #include - #include - diff --git a/depends/patches/qt/fix_montery_include.patch b/depends/patches/qt/fix_montery_include.patch new file mode 100644 index 000000000000..38b700addfe7 --- /dev/null +++ b/depends/patches/qt/fix_montery_include.patch @@ -0,0 +1,21 @@ +From dece6f5840463ae2ddf927d65eb1b3680e34a547 +From: Øystein Heskestad +Date: Wed, 27 Oct 2021 13:07:46 +0200 +Subject: [PATCH] Add missing macOS header file that was indirectly included before + +See: https://bugreports.qt.io/browse/QTBUG-97855 + +Upstream Commits: + - Qt 6.2: c884bf138a21dd7320e35cef34d24e22e74d7ce0 + +diff --git a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +index e070ba97..07c75b04 100644 +--- a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h ++++ b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +@@ -40,6 +40,7 @@ + #ifndef QIOSURFACEGRAPHICSBUFFER_H + #define QIOSURFACEGRAPHICSBUFFER_H + ++#include + #include + #include diff --git a/depends/patches/qt/fix_no_printer.patch b/depends/patches/qt/fix_no_printer.patch deleted file mode 100644 index 13723561384d..000000000000 --- a/depends/patches/qt/fix_no_printer.patch +++ /dev/null @@ -1,19 +0,0 @@ ---- x/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -+++ y/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -@@ -52,6 +52,7 @@ - // - - #include -+#include - - #ifndef QT_NO_PRINTER - ---- x/qtbase/src/plugins/plugins.pro -+++ y/qtbase/src/plugins/plugins.pro -@@ -9,6 +9,3 @@ qtHaveModule(gui) { - !android:qtConfig(library): SUBDIRS *= generic - } - qtHaveModule(widgets): SUBDIRS += styles -- --!winrt:qtHaveModule(printsupport): \ -- SUBDIRS += printsupport diff --git a/depends/patches/qt/fix_qt_pkgconfig.patch b/depends/patches/qt/fix_qt_pkgconfig.patch index a5de2b4b9ee3..73f4d89f7354 100644 --- a/depends/patches/qt/fix_qt_pkgconfig.patch +++ b/depends/patches/qt/fix_qt_pkgconfig.patch @@ -4,20 +4,8 @@ load(qt_targets) # this builds on top of qt_common --!internal_module:if(unix|mingw) { +-!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { +if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { CONFIG += create_pc QMAKE_PKGCONFIG_DESTDIR = pkgconfig host_build: \ -@@ -284,9 +284,9 @@ load(qt_targets) - QMAKE_PKGCONFIG_CFLAGS = -D$$MODULE_DEFINE -I${includedir}/$$MODULE_INCNAME - } - QMAKE_PKGCONFIG_NAME = $$replace(TARGET, ^Qt, "Qt$$QT_MAJOR_VERSION ") -- QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION) -+ QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION)$$qtPlatformTargetSuffix() - for(i, MODULE_DEPENDS): \ -- QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0)) -+ QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0))$$qtPlatformTargetSuffix() - isEmpty(QMAKE_PKGCONFIG_DESCRIPTION): \ - QMAKE_PKGCONFIG_DESCRIPTION = $$replace(TARGET, ^Qt, "Qt ") module - !isEmpty(lib_replace0.match) { diff --git a/depends/patches/qt/guix_cross_lib_path.patch b/depends/patches/qt/guix_cross_lib_path.patch new file mode 100644 index 000000000000..7911dc21d7db --- /dev/null +++ b/depends/patches/qt/guix_cross_lib_path.patch @@ -0,0 +1,17 @@ +Facilitate guix building with CROSS_LIBRARY_PATH + +See discussion in https://github.com/bitcoin/bitcoin/pull/15277. + +--- a/qtbase/mkspecs/features/toolchain.prf ++++ b/qtbase/mkspecs/features/toolchain.prf +@@ -236,8 +236,8 @@ isEmpty($${target_prefix}.INCDIRS) { + add_libraries = false + for (line, output) { + line ~= s/^[ \\t]*// # remove leading spaces +- contains(line, "LIBRARY_PATH=.*") { +- line ~= s/^LIBRARY_PATH=// # remove leading LIBRARY_PATH= ++ contains(line, "(CROSS_)?LIBRARY_PATH=.*") { ++ line ~= s/^(CROSS_)?LIBRARY_PATH=// # remove leading (CROSS_)?LIBRARY_PATH= + equals(QMAKE_HOST.os, Windows): \ + paths = $$split(line, ;) + else: \ diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 190ab7a160e4..cb94bf07b42d 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -13,13 +13,10 @@ QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} QMAKE_MAC_SDK.macosx.platform_name = macosx QMAKE_MAC_SDK.macosx.SDKVersion = $${MAC_SDK_VERSION} QMAKE_MAC_SDK.macosx.PlatformPath = /phony -QMAKE_APPLE_DEVICE_ARCHS=x86_64 !host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS -!host_build: QMAKE_CXXFLAGS += $$QMAKE_CFLAGS +!host_build: QMAKE_CXXFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_LFLAGS += -target $${MAC_TARGET} QMAKE_AR = $${CROSS_COMPILE}ar cq QMAKE_RANLIB=$${CROSS_COMPILE}ranlib -QMAKE_LIBTOOL=$${CROSS_COMPILE}libtool -QMAKE_INSTALL_NAME_TOOL=$${CROSS_COMPILE}install_name_tool load(qt_config) diff --git a/depends/patches/qt/no-xlib.patch b/depends/patches/qt/no-xlib.patch index f4a6f09ee49b..d6846aaca2c2 100644 --- a/depends/patches/qt/no-xlib.patch +++ b/depends/patches/qt/no-xlib.patch @@ -22,7 +22,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 #include #include -@@ -391,6 +393,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window) +@@ -391,6 +391,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window) xcb_flush(xcb_connection()); } @@ -30,7 +30,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 static int cursorIdForShape(int cshape) { int cursorId = 0; -@@ -444,6 +447,7 @@ static int cursorIdForShape(int cshape) +@@ -444,6 +445,7 @@ static int cursorIdForShape(int cshape) } return cursorId; } @@ -38,7 +38,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape) { -@@ -556,7 +560,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) +@@ -556,7 +558,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) xcb_cursor_t QXcbCursor::createFontCursor(int cshape) { xcb_connection_t *conn = xcb_connection(); @@ -48,16 +48,15 @@ index 7c62c2e2b3..c05c6c0a07 100644 xcb_cursor_t cursor = XCB_NONE; // Try Xcursor first -@@ -585,7 +591,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) - +@@ -586,6 +590,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) // Non-standard X11 cursors are created from bitmaps cursor = createNonStandardCursor(cshape); -- + +#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) // Create a glpyh cursor if everything else failed if (!cursor && cursorId) { cursor = xcb_generate_id(conn); -@@ -593,6 +599,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) +@@ -593,6 +598,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) cursorId, cursorId + 1, 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0); } diff --git a/depends/patches/qt/no_sdk_version_check.patch b/depends/patches/qt/no_sdk_version_check.patch deleted file mode 100644 index b16635b57202..000000000000 --- a/depends/patches/qt/no_sdk_version_check.patch +++ /dev/null @@ -1,20 +0,0 @@ -commit f5eb142cd04be2bc4ca610ed3b5b7e8ce3520ee3 -Author: fanquake -Date: Tue Jan 5 16:08:49 2021 +0800 - - Don't invoke macOS SDK version checking - - This tries to use xcrun which is not available when cross-compiling. - -diff --git a/qtbase/mkspecs/features/mac/default_post.prf b/qtbase/mkspecs/features/mac/default_post.prf -index 92a9112bca6..447e186eb26 100644 ---- a/qtbase/mkspecs/features/mac/default_post.prf -+++ b/qtbase/mkspecs/features/mac/default_post.prf -@@ -8,7 +8,6 @@ contains(TEMPLATE, .*app) { - !macx-xcode:if(isEmpty(BUILDS)|build_pass) { - # Detect changes to the platform SDK - QMAKE_EXTRA_VARIABLES += QMAKE_MAC_SDK QMAKE_MAC_SDK_VERSION QMAKE_XCODE_DEVELOPER_PATH -- QMAKE_EXTRA_INCLUDES += $$shell_quote($$PWD/sdk.mk) - } - - # Detect incompatible SDK versions diff --git a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch index 0358bea6e940..f0c14a9400e1 100644 --- a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch +++ b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch @@ -7,7 +7,7 @@ Upstream report: https://bugreports.qt.io/browse/QTBUG-83160 diff --git a/qtbase/src/tools/moc/main.cpp b/qtbase/src/tools/moc/main.cpp --- a/qtbase/src/tools/moc/main.cpp +++ b/qtbase/src/tools/moc/main.cpp -@@ -188,6 +188,7 @@ int runMoc(int argc, char **argv) +@@ -238,6 +238,7 @@ int runMoc(int argc, char **argv) dummyVariadicFunctionMacro.arguments += Symbol(0, PP_IDENTIFIER, "__VA_ARGS__"); pp.macros["__attribute__"] = dummyVariadicFunctionMacro; pp.macros["__declspec"] = dummyVariadicFunctionMacro; diff --git a/depends/patches/qt/rcc_hardcode_timestamp.patch b/depends/patches/qt/rcc_hardcode_timestamp.patch new file mode 100644 index 000000000000..03f389797564 --- /dev/null +++ b/depends/patches/qt/rcc_hardcode_timestamp.patch @@ -0,0 +1,24 @@ +Hardcode last modified timestamp in Qt RCC + +This change allows the already built qt package to be reused even with +the SOURCE_DATE_EPOCH variable set, e.g., for Guix builds. + + +--- old/qtbase/src/tools/rcc/rcc.cpp ++++ new/qtbase/src/tools/rcc/rcc.cpp +@@ -227,14 +227,7 @@ void RCCFileInfo::writeDataInfo(RCCResourceLibrary &lib) + + if (lib.formatVersion() >= 2) { + // last modified time stamp +- const QDateTime lastModified = m_fileInfo.lastModified(); +- quint64 lastmod = quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0); +- static const quint64 sourceDate = 1000 * qgetenv("QT_RCC_SOURCE_DATE_OVERRIDE").toULongLong(); +- if (sourceDate != 0) +- lastmod = sourceDate; +- static const quint64 sourceDate2 = 1000 * qgetenv("SOURCE_DATE_EPOCH").toULongLong(); +- if (sourceDate2 != 0) +- lastmod = sourceDate2; ++ quint64 lastmod = quint64(1); + lib.writeNumber8(lastmod); + if (text || pass1) + lib.writeChar('\n'); diff --git a/depends/patches/qt/use_android_ndk23.patch b/depends/patches/qt/use_android_ndk23.patch new file mode 100644 index 000000000000..f22367d527be --- /dev/null +++ b/depends/patches/qt/use_android_ndk23.patch @@ -0,0 +1,13 @@ +Use Android NDK r23 LTS + +--- old/qtbase/mkspecs/features/android/default_pre.prf ++++ new/qtbase/mkspecs/features/android/default_pre.prf +@@ -76,7 +76,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- + else: equals(QT_ARCH, arm64-v8a): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/aarch64-linux-android- + else: CROSS_COMPILE = $$NDK_LLVM_PATH/bin/arm-linux-androideabi- + +-QMAKE_RANLIB = $${CROSS_COMPILE}ranlib ++QMAKE_RANLIB = $$NDK_LLVM_PATH/bin/llvm-ranlib + QMAKE_LINK_SHLIB = $$QMAKE_LINK + QMAKE_LFLAGS = + diff --git a/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch b/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch new file mode 100644 index 000000000000..c115bc43e81f --- /dev/null +++ b/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch @@ -0,0 +1,27 @@ +Remove _SDT_ASM_SECTION_AUTOGROUP_SUPPORT check + +We assume that the assembler supports "?" in .pushsection directives. +This enables us to skip configure and make. + +See https://github.com/bitcoin/bitcoin/issues/23297. + +diff --git a/includes/sys/sdt.h b/includes/sys/sdt.h +index ca0162b..f96e0ee 100644 +--- a/includes/sys/sdt.h ++++ b/includes/sys/sdt.h +@@ -241,12 +241,10 @@ __extension__ extern unsigned long long __sdt_unsp; + nice with code in COMDAT sections, which comes up in C++ code. + Without that assembler support, some combinations of probe placements + in certain kinds of C++ code may produce link-time errors. */ +-#include "sdt-config.h" +-#if _SDT_ASM_SECTION_AUTOGROUP_SUPPORT ++/* PATCH: We assume that the assembler supports the feature. This ++ enables us to skip configure and make. In turn, this means we ++ require fewer dependencies and have shorter depend build times. */ + # define _SDT_ASM_AUTOGROUP "?" +-#else +-# define _SDT_ASM_AUTOGROUP "" +-#endif + + #define _SDT_DEF_MACROS \ + _SDT_ASM_1(.altmacro) \ diff --git a/depends/patches/zeromq/netbsd_kevent_void.patch b/depends/patches/zeromq/netbsd_kevent_void.patch new file mode 100644 index 000000000000..845c6bdda671 --- /dev/null +++ b/depends/patches/zeromq/netbsd_kevent_void.patch @@ -0,0 +1,57 @@ +commit 129137d5182967dbfcfec66bad843df2a992a78f +Author: fanquake +Date: Mon Jan 3 20:13:33 2022 +0800 + + problem: kevent udata is now void* on NetBSD Current (10) + + solution: check for the intptr_t variant in configure. + +diff --git a/configure.ac b/configure.ac +index 1a571291..402f8b86 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -307,6 +307,27 @@ case "${host_os}" in + if test "x$libzmq_netbsd_has_atomic" = "xno"; then + AC_DEFINE(ZMQ_FORCE_MUTEXES, 1, [Force to use mutexes]) + fi ++ # NetBSD Current (to become 10) has changed the type of udata in it's ++ # kevent struct from intptr_t to void * to align with darwin and other ++ # BSDs, see upstream commit: ++ # https://github.com/NetBSD/src/commit/e5ead823eb916b56589d2c6c560dbcfe4a2d0afc ++ AC_MSG_CHECKING([whether kevent udata type is intptr_t]) ++ AC_LANG_PUSH([C++]) ++ AC_LINK_IFELSE([AC_LANG_PROGRAM( ++ [[#include ++ #include ++ #include ]], ++ [[struct kevent ev; ++ intptr_t udata; ++ EV_SET(&ev, 0, 0, EV_ADD, 0, 0, udata); ++ return 0;]])], ++ [libzmq_netbsd_kevent_udata_intptr_t=yes], ++ [libzmq_netbsd_kevent_udata_intptr_t=no]) ++ AC_LANG_POP([C++]) ++ AC_MSG_RESULT([$libzmq_netbsd_kevent_udata_intptr_t]) ++ if test "x$libzmq_netbsd_kevent_udata_intptr_t" = "xyes"; then ++ AC_DEFINE(ZMQ_NETBSD_KEVENT_UDATA_INTPTR_T, 1, [kevent udata type is intptr_t]) ++ fi + ;; + *openbsd*|*bitrig*) + # Define on OpenBSD to enable all library features +diff --git a/src/kqueue.cpp b/src/kqueue.cpp +index 53d82ac4..a6a7a7f2 100644 +--- a/src/kqueue.cpp ++++ b/src/kqueue.cpp +@@ -46,9 +46,9 @@ + #include "i_poll_events.hpp" + #include "likely.hpp" + +-// NetBSD defines (struct kevent).udata as intptr_t, everyone else +-// as void *. +-#if defined ZMQ_HAVE_NETBSD ++// NetBSD up to version 9 defines (struct kevent).udata as intptr_t, ++// everyone else as void *. ++#if defined ZMQ_HAVE_NETBSD && defined(ZMQ_NETBSD_KEVENT_UDATA_INTPTR_T) + #define kevent_udata_t intptr_t + #else + #define kevent_udata_t void * diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 21bf587eaf31..d8fd46d1c7d0 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -863,9 +863,7 @@ RECURSIVE = YES EXCLUDE = src/crc32c \ src/leveldb \ - src/json \ - src/test \ - src/qt/test + src/json # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index 12807bfb8650..ab5db58cdd18 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -5,6 +5,28 @@ The headless daemon `bitcoind` has the JSON-RPC API enabled by default, the GUI option. In the GUI it is possible to execute RPC methods in the Debug Console Dialog. +## Parameter passing + +The JSON-RPC server supports both _by-position_ and _by-name_ [parameter +structures](https://www.jsonrpc.org/specification#parameter_structures) +described in the JSON-RPC specification. For extra convenience, to avoid the +need to name every parameter value, all RPC methods accept a named parameter +called `args`, which can be set to an array of initial positional values that +are combined with named values. + +Examples: + +```sh +# "params": ["mywallet", false, false, "", false, false, true] +bitcoin-cli createwallet mywallet false false "" false false true + +# "params": {"wallet_name": "mywallet", "load_on_startup": true} +bitcoin-cli -named createwallet wallet_name=mywallet load_on_startup=true + +# "params": {"args": ["mywallet"], "load_on_startup": true} +bitcoin-cli -named createwallet mywallet load_on_startup=true +``` + ## Versioning The RPC interface might change from one major version of Bitcoin Core to the diff --git a/doc/README.md b/doc/README.md index 38f6b1d3275c..c570432aa4d7 100644 --- a/doc/README.md +++ b/doc/README.md @@ -46,7 +46,6 @@ The following are developer notes on how to build Bitcoin Core on your native pl - [OpenBSD Build Notes](build-openbsd.md) - [NetBSD Build Notes](build-netbsd.md) - [Android Build Notes](build-android.md) -- [Gitian Building Guide (External Link)](https://github.com/bitcoin-core/docs/blob/master/gitian-building.md) Development --------------------- @@ -54,7 +53,6 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Developer Notes](developer-notes.md) - [Productivity Notes](productivity.md) -- [Release Notes](release-notes.md) - [Release Process](release-process.md) - [Source Code Documentation (External Link)](https://doxygen.bitcoincore.org/) - [Translation Process](translation_process.md) @@ -65,6 +63,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [BIPS](bips.md) - [Dnsseed Policy](dnsseed-policy.md) - [Benchmarking](benchmarking.md) +- [Internal Design Docs](design/) ### Resources * Discuss on the [BitcoinTalk](https://bitcointalk.org/) forums, in the [Development & Technical Discussion board](https://bitcointalk.org/index.php?board=6.0). @@ -73,14 +72,19 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th ### Miscellaneous - [Assets Attribution](assets-attribution.md) - [bitcoin.conf Configuration File](bitcoin-conf.md) +- [CJDNS Support](cjdns.md) - [Files](files.md) - [Fuzz-testing](fuzzing.md) - [I2P Support](i2p.md) - [Init Scripts (systemd/upstart/openrc)](init.md) +- [Managing Wallets](managing-wallets.md) +- [Multisig Tutorial](multisig-tutorial.md) +- [P2P bad ports definition and list](p2p-bad-ports.md) - [PSBT support](psbt.md) - [Reduce Memory](reduce-memory.md) - [Reduce Traffic](reduce-traffic.md) - [Tor Support](tor.md) +- [Transaction Relay Policy](policy/README.md) - [ZMQ](zmq.md) License diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 8b281acca773..a704b969df8a 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -31,13 +31,14 @@ Supported API `GET /rest/tx/.` Given a transaction hash: returns a transaction in binary, hex-encoded binary, or JSON formats. +Responds with 404 if the transaction doesn't exist. By default, this endpoint will only search the mempool. To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. #### Blocks -`GET /rest/block/.` -`GET /rest/block/notxdetails/.` +- `GET /rest/block/.` +- `GET /rest/block/notxdetails/.` Given a block hash: returns a block, in binary, hex-encoded binary or JSON formats. Responds with 404 if the block doesn't exist. @@ -47,39 +48,61 @@ The HTTP request and response are both handled entirely in-memory. With the /notxdetails/ option JSON response will only contain the transaction hash instead of the complete transaction details. The option only affects the JSON response. #### Blockheaders -`GET /rest/headers//.` +`GET /rest/headers/.?count=` Given a block hash: returns amount of blockheaders in upward direction. Returns empty if the block doesn't exist or it isn't in the active chain. +*Deprecated (but not removed) since 24.0:* +`GET /rest/headers//.` + +#### Blockfilter Headers +`GET /rest/blockfilterheaders//.?count=` + +Given a block hash: returns amount of blockfilter headers in upward +direction for the filter type . +Returns empty if the block doesn't exist or it isn't in the active chain. + +*Deprecated (but not removed) since 24.0:* +`GET /rest/blockfilterheaders///.` + +#### Blockfilters +`GET /rest/blockfilter//.` + +Given a block hash: returns the block filter of the given block of type +. +Responds with 404 if the block doesn't exist. + #### Blockhash by height `GET /rest/blockhashbyheight/.` Given a height: returns hash of block in best-block-chain at height provided. +Responds with 404 if block not found. #### Chaininfos `GET /rest/chaininfo.json` Returns various state info regarding block chain processing. Only supports JSON as output format. -* chain : (string) current network name (main, test, signet, regtest) -* blocks : (numeric) the current number of blocks processed in the server -* headers : (numeric) the current number of headers we have validated -* bestblockhash : (string) the hash of the currently best block -* difficulty : (numeric) the current difficulty -* mediantime : (numeric) the median time of the 11 blocks before the most recent block on the blockchain -* verificationprogress : (numeric) estimate of verification progress [0..1] -* chainwork : (string) total amount of work in active chain, in hexadecimal -* pruned : (boolean) if the blocks are subject to pruning -* pruneheight : (numeric) highest block available -* softforks : (array) status of softforks in progress +Refer to the `getblockchaininfo` RPC help for details. + +#### Deployment info +`GET /rest/deploymentinfo.json` +`GET /rest/deploymentinfo/.json` + +Returns an object containing various state info regarding deployments of +consensus changes at the current chain tip, or at if provided. +Only supports JSON as output format. +Refer to the `getdeploymentinfo` RPC help for details. #### Query UTXO set -`GET /rest/getutxos//-/-/.../-.` +- `GET /rest/getutxos/-/-/.../-.` +- `GET /rest/getutxos/checkmempool/-/-/.../-.` -The getutxo command allows querying of the UTXO set given a set of outpoints. -See BIP64 for input and output serialisation: -https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki +The getutxos endpoint allows querying the UTXO set, given a set of outpoints. +With the `/checkmempool/` option, the mempool is also taken into account. +See [BIP64](https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki) for +input and output serialization (relevant for `bin` and `hex` output formats). Example: ``` @@ -94,6 +117,7 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 "value" : 8.8687, "scriptPubKey" : { "asm" : "OP_DUP OP_HASH160 1c7cebb529b86a04c683dfa87be49de35bcf589e OP_EQUALVERIFY OP_CHECKSIG", + "desc" : "addr(mi7as51dvLJsizWnTMurtRmrP8hG2m1XvD)#gj9tznmy" "hex" : "76a9141c7cebb529b86a04c683dfa87be49de35bcf589e88ac", "type" : "pubkeyhash", "address" : "mi7as51dvLJsizWnTMurtRmrP8hG2m1XvD" @@ -106,14 +130,15 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 #### Memory pool `GET /rest/mempool/info.json` -Returns various information about the TX mempool. +Returns various information about the transaction mempool. Only supports JSON as output format. -Refer to the `getmempoolinfo` RPC for documentation of the fields. +Refer to the `getmempoolinfo` RPC help for details. `GET /rest/mempool/contents.json` -Returns transactions in the TX mempool. +Returns the transactions in the mempool. Only supports JSON as output format. +Refer to the `getrawmempool` RPC help for details. Risks ------------- diff --git a/doc/bips.md b/doc/bips.md index 45954bcfd8f0..1d5c91b8bdb3 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,4 +1,4 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**): +BIPs that are implemented by Bitcoin Core (up-to-date up to **v24.0**): * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1** ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575)) * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). @@ -28,11 +28,12 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**): and it is disabled by default at build time since **v0.19.0** ([PR #15584](https://github.com/bitcoin/bitcoin/pull/15584)). It has been removed as of **v0.20.0** ([PR 17165](https://github.com/bitcoin/bitcoin/pull/17165)). * [`BIP 84`](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki): The experimental descriptor wallets introduced in **v0.21.0** by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 84. ([PR #16528](https://github.com/bitcoin/bitcoin/pull/16528)) +* [`BIP 86`](https://github.com/bitcoin/bips/blob/master/bip-0086.mediawiki): Descriptor wallets by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 86 since **v23.0** ([PR #22364](https://github.com/bitcoin/bitcoin/pull/22364)). * [`BIP 90`](https://github.com/bitcoin/bips/blob/master/bip-0090.mediawiki): Trigger mechanism for activation of BIPs 34, 65, and 66 has been simplified to block height checks since **v0.14.0** ([PR #8391](https://github.com/bitcoin/bitcoin/pull/8391)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). * [`BIP 113`](https://github.com/bitcoin/bips/blob/master/bip-0113.mediawiki): Median time past lock-time calculations have been implemented since **v0.12.1** ([PR #6566](https://github.com/bitcoin/bitcoin/pull/6566)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). -* [`BIP 125`](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki): Opt-in full replace-by-fee signaling honoured in mempool and mining as of **v0.12.0** ([PR 6871](https://github.com/bitcoin/bitcoin/pull/6871)). Enabled by default in the wallet GUI as of **v0.18.1** ([PR #11605](https://github.com/bitcoin/bitcoin/pull/11605)) +* [`BIP 125`](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki): Opt-in full replace-by-fee signaling partially implemented. See doc/policy/mempool-replacements.md. * [`BIP 130`](https://github.com/bitcoin/bips/blob/master/bip-0130.mediawiki): direct headers announcement is negotiated with peer versions `>=70012` as of **v0.12.0** ([PR 6494](https://github.com/bitcoin/bitcoin/pull/6494)). * [`BIP 133`](https://github.com/bitcoin/bips/blob/master/bip-0133.mediawiki): feefilter messages are respected and sent for peer versions `>=70013` as of **v0.13.0** ([PR 7542](https://github.com/bitcoin/bitcoin/pull/7542)). * [`BIP 141`](https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki): Segregated Witness (Consensus Layer) as of **v0.13.0** ([PR 8149](https://github.com/bitcoin/bitcoin/pull/8149)), defined for mainnet as of **v0.13.1** ([PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)), and *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). @@ -42,7 +43,8 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**): * [`BIP 147`](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki): NULLDUMMY softfork as of **v0.13.1** ([PR 8636](https://github.com/bitcoin/bitcoin/pull/8636) and [PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)), *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). * [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)). * [`BIP 155`](https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki): The 'addrv2' and 'sendaddrv2' messages which enable relay of Tor V3 addresses (and other networks) are supported as of **v0.21.0** ([PR 19954](https://github.com/bitcoin/bitcoin/pull/19954)). -* [`BIP 158`](https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki): Compact Block Filters for Light Clients can be indexed as of **v0.19.0** ([PR #14121](https://github.com/bitcoin/bitcoin/pull/14121)). +* [`BIP 157`](https://github.com/bitcoin/bips/blob/master/bip-0157.mediawiki) + [`158`](https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki): Compact Block Filters for Light Clients can be indexed as of **v0.19.0** ([PR #14121](https://github.com/bitcoin/bitcoin/pull/14121)) and served to peers on the P2P network as of **v0.21.0** ([PR #16442](https://github.com/bitcoin/bitcoin/pull/16442)). * [`BIP 159`](https://github.com/bitcoin/bips/blob/master/bip-0159.mediawiki): The `NODE_NETWORK_LIMITED` service bit is signalled as of **v0.16.0** ([PR 11740](https://github.com/bitcoin/bitcoin/pull/11740)), and such nodes are connected to as of **v0.17.0** ([PR 10387](https://github.com/bitcoin/bitcoin/pull/10387)). * [`BIP 173`](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki): Bech32 addresses for native Segregated Witness outputs are supported as of **v0.16.0** ([PR 11167](https://github.com/bitcoin/bitcoin/pull/11167)). Bech32 addresses are generated by default as of **v0.20.0** ([PR 16884](https://github.com/bitcoin/bitcoin/pull/16884)). * [`BIP 174`](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki): RPCs to operate on Partially Signed Bitcoin Transactions (PSBT) are present as of **v0.17.0** ([PR 13557](https://github.com/bitcoin/bitcoin/pull/13557)). @@ -57,3 +59,12 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**): with mainnet activation as of **v0.21.1** ([PR 21377](https://github.com/bitcoin/bitcoin/pull/21377), [PR 21686](https://github.com/bitcoin/bitcoin/pull/21686)). * [`BIP 350`](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki): Addresses for native v1+ segregated Witness outputs use Bech32m instead of Bech32 as of **v22.0** ([PR 20861](https://github.com/bitcoin/bitcoin/pull/20861)). +* [`BIP 371`](https://github.com/bitcoin/bips/blob/master/bip-0371.mediawiki): Taproot fields for PSBT as of **v24.0** ([PR 22558](https://github.com/bitcoin/bitcoin/pull/22558)). +* [`BIP 380`](https://github.com/bitcoin/bips/blob/master/bip-0380.mediawiki) + [`381`](https://github.com/bitcoin/bips/blob/master/bip-0381.mediawiki) + [`382`](https://github.com/bitcoin/bips/blob/master/bip-0382.mediawiki) + [`383`](https://github.com/bitcoin/bips/blob/master/bip-0383.mediawiki) + [`384`](https://github.com/bitcoin/bips/blob/master/bip-0384.mediawiki) + [`385`](https://github.com/bitcoin/bips/blob/master/bip-0385.mediawiki): + Output Script Descriptors, and most of Script Expressions are implemented as of **v0.17.0** ([PR 13697](https://github.com/bitcoin/bitcoin/pull/13697)). +* [`BIP 386`](https://github.com/bitcoin/bips/blob/master/bip-0386.mediawiki): tr() Output Script Descriptors are implemented as of **v22.0** ([PR 22051](https://github.com/bitcoin/bitcoin/pull/22051)). diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index 8c9035c45b6f..1ebfb4c1de94 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -6,6 +6,8 @@ All command-line options (except for `-?`, `-help`, `-version` and `-conf`) may Changes to the configuration file while `bitcoind` or `bitcoin-qt` is running only take effect after restarting. +Users should never make any configuration changes which they do not understand. Furthermore, users should always be wary of accepting any configuration changes provided to them by another source (even if they believe that they do understand them). + ## Configuration File Format The configuration file is a plain text file and consists of `option=value` entries, one per line. Leading and trailing whitespaces are removed. @@ -61,4 +63,12 @@ Windows | `%APPDATA%\Bitcoin\` | `C:\Users\username\AppData\Roaming\Bitcoin\bitc Linux | `$HOME/.bitcoin/` | `/home/username/.bitcoin/bitcoin.conf` macOS | `$HOME/Library/Application Support/Bitcoin/` | `/Users/username/Library/Application Support/Bitcoin/bitcoin.conf` -You can find an example bitcoin.conf file in [share/examples/bitcoin.conf](../share/examples/bitcoin.conf). +An example configuration file can be generated by [contrib/devtools/gen-bitcoin-conf.sh](../contrib/devtools/gen-bitcoin-conf.sh). +Run this script after compiling to generate an up-to-date configuration file. +The output is placed under `share/examples/bitcoin.conf`. +To use the generated configuration file, copy the example file into your data directory and edit it there, like so: + +``` +# example copy command for linux user +cp share/examples/bitcoin.conf ~/.bitcoin +``` diff --git a/doc/build-android.md b/doc/build-android.md index 7a8a9e6a653e..2f2e01c441dc 100644 --- a/doc/build-android.md +++ b/doc/build-android.md @@ -3,9 +3,22 @@ ANDROID BUILD NOTES This guide describes how to build and package the `bitcoin-qt` GUI for Android on Linux and macOS. -## Preparation -You will need to get the Android NDK and build dependencies for Android as described in [depends/README.md](../depends/README.md). +## Dependencies + +Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). + +The minimum supported Android NDK version is [r23](https://github.com/android/ndk/wiki/Changelog-r23). + +In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. + +API levels from 24 to 29 have been tested to work. + +If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. +This is an example command for a default build with no disabled dependencies: + + ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin + ## Building and packaging diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index da2ab61c2a72..a8e643a2ab0e 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -1,48 +1,21 @@ # FreeBSD Build Guide -**Updated for FreeBSD [12.2](https://www.freebsd.org/releases/12.2R/announce.html)** +**Updated for FreeBSD [12.3](https://www.freebsd.org/releases/12.3R/announce/)** This guide describes how to build bitcoind, command-line utilities, and GUI on FreeBSD. -## Dependencies - -The following dependencies are **required**: - - Library | Purpose | Description - ----------------------------------------------------------------------|------------|---------------------- - [autoconf](https://svnweb.freebsd.org/ports/head/devel/autoconf/) | Build | Automatically configure software source code - [automake](https://svnweb.freebsd.org/ports/head/devel/automake/) | Build | Generate makefile (requires autoconf) - [libtool](https://svnweb.freebsd.org/ports/head/devel/libtool/) | Build | Shared library support - [pkgconf](https://svnweb.freebsd.org/ports/head/devel/pkgconf/) | Build | Configure compiler and linker flags - [git](https://svnweb.freebsd.org/ports/head/devel/git/) | Clone | Version control system - [gmake](https://svnweb.freebsd.org/ports/head/devel/gmake/) | Compile | Generate executables - [boost-libs](https://svnweb.freebsd.org/ports/head/devel/boost-libs/) | Utility | Library for threading, data structures, etc - [libevent](https://svnweb.freebsd.org/ports/head/devel/libevent/) | Networking | OS independent asynchronous networking - - -The following dependencies are **optional**: - - Library | Purpose | Description - ---------------------------------------------------------------------------|------------------|---------------------- - [db5](https://svnweb.freebsd.org/ports/head/databases/db5/) | Berkeley DB | Wallet storage (only needed when wallet enabled) - [qt5](https://svnweb.freebsd.org/ports/head/devel/qt5/) | GUI | GUI toolkit (only needed when GUI enabled) - [libqrencode](https://svnweb.freebsd.org/ports/head/graphics/libqrencode/) | QR codes in GUI | Generating QR codes (only needed when GUI enabled) - [libzmq4](https://svnweb.freebsd.org/ports/head/net/libzmq4/) | ZMQ notification | Allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) - [sqlite3](https://svnweb.freebsd.org/ports/head/databases/sqlite3/) | SQLite DB | Wallet storage (only needed when wallet enabled) - [python3](https://svnweb.freebsd.org/ports/head/lang/python3/) | Testing | Python Interpreter (only needed when running the test suite) - - See [dependencies.md](dependencies.md) for a complete overview. - ## Preparation ### 1. Install Required Dependencies -Install the required dependencies the usual way you [install software on FreeBSD](https://www.freebsd.org/doc/en/books/handbook/ports.html) - either with `pkg` or via the Ports collection. The example commands below use `pkg` which is usually run as `root` or via `sudo`. If you want to use `sudo`, and you haven't set it up: [use this guide](http://www.freebsdwiki.net/index.php/Sudo%2C_configuring) to setup `sudo` access on FreeBSD. +Run the following as root to install the base dependencies for building. ```bash pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf ``` +See [dependencies.md](dependencies.md) for a complete overview. + ### 2. Clone Bitcoin Repo Now that `git` and all the required dependencies are installed, let's clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. ``` bash @@ -52,21 +25,23 @@ git clone https://github.com/bitcoin/bitcoin.git ### 3. Install Optional Dependencies #### Wallet Dependencies -It is not necessary to build wallet functionality to run bitcoind or the GUI. To enable legacy wallets, you must install `db5`. To enable [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md), `sqlite3` is required. Skip `db5` if you intend to *exclusively* use descriptor wallets - -###### Legacy Wallet Support -`db5` is required to enable support for legacy wallets. Skip if you don't intend to use legacy wallets - -```bash -pkg install db5 -``` +It is not necessary to build wallet functionality to run either `bitcoind` or `bitcoin-qt`. ###### Descriptor Wallet Support -`sqlite3` is required to enable support for descriptor wallets. Skip if you don't intend to use descriptor wallets. +`sqlite3` is required to support [descriptor wallets](descriptors.md). +Skip if you don't intend to use descriptor wallets. ``` bash pkg install sqlite3 ``` + +###### Legacy Wallet Support +`db5` is only required to support legacy wallets. +Skip if you don't intend to use legacy wallets. + +```bash +pkg install db5 +``` --- #### GUI Dependencies @@ -84,6 +59,14 @@ pkg install libqrencode ``` --- +#### Notifications +###### ZeroMQ + +Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +```bash +pkg install libzmq4 +``` + #### Test Suite Dependencies There is an included test suite that is useful for testing code changes when developing. To run the test suite (recommended), you will need to have Python 3 installed: @@ -98,8 +81,17 @@ pkg install python3 ### 1. Configuration There are many ways to configure Bitcoin Core, here are a few common examples: -##### Wallet (BDB + SQlite) Support, No GUI: -This explicitly enables legacy wallet support and disables the GUI. If `sqlite3` is installed, then descriptor wallet support will be built. + +##### Descriptor Wallet and GUI: +This explicitly enables the GUI and disables legacy wallet support, assuming `sqlite` and `qt` are installed. +```bash +./autogen.sh +./configure --without-bdb --with-gui=yes MAKE=gmake +``` + +##### Descriptor & Legacy Wallet. No GUI: +This enables support for both wallet types and disables the GUI, assuming +`sqlite3` and `db5` are both installed. ```bash ./autogen.sh ./configure --with-gui=no --with-incompatible-bdb \ @@ -108,12 +100,6 @@ This explicitly enables legacy wallet support and disables the GUI. If `sqlite3` MAKE=gmake ``` -##### Wallet (only SQlite) and GUI Support: -This explicitly enables the GUI and disables legacy wallet support. If `qt5` is not installed, this will throw an error. If `sqlite3` is installed then descriptor wallet functionality will be built. If `sqlite3` is not installed, then wallet functionality will be disabled. -```bash -./autogen.sh -./configure --without-bdb --with-gui=yes MAKE=gmake -``` ##### No Wallet or GUI ``` bash ./autogen.sh diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index edabd7161178..0f05cdcba789 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -1,81 +1,116 @@ -NetBSD build guide -====================== -(updated for NetBSD 8.0) +# NetBSD Build Guide -This guide describes how to build bitcoind and command-line utilities on NetBSD. +Updated for NetBSD [9.2](https://netbsd.org/releases/formal-9/NetBSD-9.2.html). -This guide does not contain instructions for building the GUI. +This guide describes how to build bitcoind, command-line utilities, and GUI on NetBSD. -Preparation -------------- +## Preparation -You will need the following modules, which can be installed via pkgsrc or pkgin: +### 1. Install Required Dependencies + +Install the required dependencies the usual way you [install software on NetBSD](https://www.netbsd.org/docs/guide/en/chap-boot.html#chap-boot-pkgsrc). +The example commands below use `pkgin`. + +```bash +pkgin install autoconf automake libtool pkg-config git gmake boost libevent ``` -autoconf -automake -boost -git -gmake -libevent -libtool -pkg-config -python37 -git clone https://github.com/bitcoin/bitcoin.git +NetBSD currently ships with an older version of `gcc` than is needed to build. You should upgrade your `gcc` and then pass this new version to the configure script. + +For example, grab `gcc9`: +``` +pkgin install gcc9 +``` + +Then, when configuring, pass the following: +```bash +./configure + ... + CC="/usr/pkg/gcc9/bin/gcc" \ + CXX="/usr/pkg/gcc9/bin/g++" \ + ... ``` See [dependencies.md](dependencies.md) for a complete overview. -### Building BerkeleyDB +### 2. Clone Bitcoin Repo + +Clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass -`--disable-wallet` to `./configure` and skip to the next section. +```bash +git clone https://github.com/bitcoin/bitcoin.git +``` + +### 3. Install Optional Dependencies + +#### Wallet Dependencies -It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports, for the same reason as boost above (g++/libstd++ incompatibility). -If you have to build it yourself, you can use [the installation script included -in contrib/](/contrib/install_db4.sh) like so: +It is not necessary to build wallet functionality to run bitcoind or the GUI. + +###### Descriptor Wallet Support + +`sqlite3` is required to enable support for [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md). ```bash -./contrib/install_db4.sh `pwd` +pkgin install sqlite3 ``` -from the root of the repository. Then set `BDB_PREFIX` for the next section: +###### Legacy Wallet Support + +`db4` is required to enable support for legacy wallets. ```bash -export BDB_PREFIX="$PWD/db4" +pkgin install db4 ``` -### Building Bitcoin Core +#### GUI Dependencies -**Important**: Use `gmake` (the non-GNU `make` will exit with an error). +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install `qt5`. -With wallet: ```bash -./autogen.sh -./configure --with-gui=no CPPFLAGS="-I/usr/pkg/include" \ - LDFLAGS="-L/usr/pkg/lib" \ - BOOST_CPPFLAGS="-I/usr/pkg/include" \ - BOOST_LDFLAGS="-L/usr/pkg/lib" \ - BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ - BDB_CFLAGS="-I${BDB_PREFIX}/include" \ - MAKE=gmake +pkgin install qt5 +``` + +The GUI can encode addresses in a QR Code. To build in QR support for the GUI, install `qrencode`. + +```bash +pkgin install qrencode ``` -Without wallet: +#### Test Suite Dependencies + +There is an included test suite that is useful for testing code changes when developing. +To run the test suite (recommended), you will need to have Python 3 installed: + +```bash +pkgin install python37 +``` + +### Building Bitcoin Core + +**Note**: Use `gmake` (the non-GNU `make` will exit with an error). + + +### 1. Configuration + +There are many ways to configure Bitcoin Core. Here is an example that +explicitly disables the wallet and GUI: + ```bash ./autogen.sh -./configure --with-gui=no --disable-wallet \ +./configure --without-wallet --with-gui=no \ CPPFLAGS="-I/usr/pkg/include" \ - LDFLAGS="-L/usr/pkg/lib" \ - BOOST_CPPFLAGS="-I/usr/pkg/include" \ - BOOST_LDFLAGS="-L/usr/pkg/lib" \ MAKE=gmake ``` +For a full list of configuration options, see the output of `./configure --help` + +### 2. Compile + Build and run the tests: + ```bash gmake # use "-j N" here for N parallel jobs -gmake check +gmake check # Run tests if Python 3 is available ``` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 89fd506f1399..afbb5c8e7584 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,123 +1,130 @@ -OpenBSD build guide -====================== -(updated for OpenBSD 6.9) +# OpenBSD Build Guide -This guide describes how to build bitcoind, bitcoin-qt, and command-line utilities on OpenBSD. +**Updated for OpenBSD [7.1](https://www.openbsd.org/71.html)** -Preparation -------------- +This guide describes how to build bitcoind, command-line utilities, and GUI on OpenBSD. -Run the following as root to install the base dependencies for building: +## Preparation + +### 1. Install Required Dependencies +Run the following as root to install the base dependencies for building. ```bash -pkg_add git gmake libevent libtool boost -pkg_add qt5 # (optional for enabling the GUI) -pkg_add autoconf # (select highest version, e.g. 2.69) -pkg_add automake # (select highest version, e.g. 1.16) -pkg_add python # (select highest version, e.g. 3.8) -pkg_add bash +pkg_add bash git gmake libevent libtool boost +# Select the newest version of the following packages: +pkg_add autoconf automake python +``` +See [dependencies.md](dependencies.md) for a complete overview. + +### 2. Clone Bitcoin Repo +Clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. +``` bash git clone https://github.com/bitcoin/bitcoin.git ``` -See [dependencies.md](dependencies.md) for a complete overview. +### 3. Install Optional Dependencies -**Important**: From OpenBSD 6.2 onwards a C++11-supporting clang compiler is -part of the base image, and while building it is necessary to make sure that -this compiler is used and not ancient g++ 4.2.1. This is done by appending -`CC=cc CC_FOR_BUILD=cc CXX=c++` to configuration commands. Mixing different -compilers within the same executable will result in errors. +#### Wallet Dependencies -### Building BerkeleyDB +It is not necessary to build wallet functionality to run either `bitcoind` or `bitcoin-qt`. + +###### Descriptor Wallet Support + +`sqlite3` is required to support [descriptor wallets](descriptors.md). + +``` bash +pkg_add sqlite3 +``` -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass -`--disable-wallet` to `./configure` and skip to the next section. +###### Legacy Wallet Support +BerkeleyDB is only required to support legacy wallets. It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports, for the same reason as boost above (g++/libstd++ incompatibility). -If you have to build it yourself, you can use [the installation script included -in contrib/](/contrib/install_db4.sh) like so: +from ports. However you can build it yourself, [using the installation script included in contrib/](/contrib/install_db4.sh), like so, from the root of the repository. ```bash -./contrib/install_db4.sh `pwd` CC=cc CXX=c++ +./contrib/install_db4.sh `pwd` ``` -from the root of the repository. Then set `BDB_PREFIX` for the next section: +Then set `BDB_PREFIX`: ```bash export BDB_PREFIX="$PWD/db4" ``` -### Building Bitcoin Core +#### GUI Dependencies +###### Qt5 + +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, Qt 5 is required. + +```bash +pkg_add qt5 +``` + +## Building Bitcoin Core **Important**: Use `gmake` (the non-GNU `make` will exit with an error). Preparation: ```bash -# Replace this with the autoconf version that you installed. Include only -# the major and minor parts of the version: use "2.69" for "autoconf-2.69p2". -export AUTOCONF_VERSION=2.69 - -# Replace this with the automake version that you installed. Include only -# the major and minor parts of the version: use "1.16" for "automake-1.16.1". +# Adapt the following for the version you installed (major.minor only): +export AUTOCONF_VERSION=2.71 export AUTOMAKE_VERSION=1.16 ./autogen.sh ``` -Make sure `BDB_PREFIX` is set to the appropriate path from the above steps. -Note that building with external signer support currently fails on OpenBSD, -hence you have to explicitely disable it by passing the parameter -`--disable-external-signer` to the configure script. -(Background: the feature requires the header-only library boost::process, which -is available on OpenBSD 6.9 via Boost 1.72.0, but contains certain system calls -and preprocessor defines like `waitid()` and `WEXITED` that are not available.) +### 1. Configuration -To configure with wallet: -```bash -./configure --with-gui=no --disable-external-signer CC=cc CXX=c++ \ - BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ - BDB_CFLAGS="-I${BDB_PREFIX}/include" \ - MAKE=gmake -``` +Note that external signer support is currently not available on OpenBSD, since +the used header-only library Boost.Process fails to compile (certain system +calls and preprocessor defines like `waitid()` and `WEXITED` are missing). + +There are many ways to configure Bitcoin Core, here are a few common examples: + +##### Descriptor Wallet and GUI: +This enables the GUI and descriptor wallet support, assuming `sqlite` and `qt5` are installed. -To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CC_FOR_BUILD=cc CXX=c++ MAKE=gmake +./configure MAKE=gmake ``` -To configure with GUI: +##### Descriptor & Legacy Wallet. No GUI: +This enables support for both wallet types and disables the GUI: + ```bash -./configure --with-gui=yes --disable-external-signer CC=cc CXX=c++ \ +./configure --with-gui=no \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake ``` -Build and run the tests: +### 2. Compile +**Important**: Use `gmake` (the non-GNU `make` will exit with an error). + ```bash -gmake # use "-j N" here for N parallel jobs -gmake check +gmake # use "-j N" for N parallel jobs +gmake check # Run tests if Python 3 is available ``` -Resource limits -------------------- +## Resource limits If the build runs into out-of-memory errors, the instructions in this section might help. The standard ulimit restrictions in OpenBSD are very strict: - - data(kbytes) 1572864 +```bash +data(kbytes) 1572864 +``` This is, unfortunately, in some cases not enough to compile some `.cpp` files in the project, (see issue [#6658](https://github.com/bitcoin/bitcoin/issues/6658)). If your user is in the `staff` group the limit can be raised with: - - ulimit -d 3000000 - +```bash +ulimit -d 3000000 +``` The change will only affect the current shell and processes spawned by it. To make the change system-wide, change `datasize-cur` and `datasize-max` in `/etc/login.conf`, and reboot. - diff --git a/doc/build-osx.md b/doc/build-osx.md index 467feff4106f..f11ed97e098a 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -4,43 +4,6 @@ This guide describes how to build bitcoind, command-line utilities, and GUI on macOS -**Note:** The following is for Intel Macs only! - -## Dependencies - -The following dependencies are **required**: - -Library | Purpose | Description ------------------------------------------------------------|------------|---------------------- -[automake](https://formulae.brew.sh/formula/automake) | Build | Generate makefile -[libtool](https://formulae.brew.sh/formula/libtool) | Build | Shared library support -[pkg-config](https://formulae.brew.sh/formula/pkg-config) | Build | Configure compiler and linker flags -[boost](https://formulae.brew.sh/formula/boost) | Utility | Library for threading, data structures, etc -[libevent](https://formulae.brew.sh/formula/libevent) | Networking | OS independent asynchronous networking - -The following dependencies are **optional**: - -Library | Purpose | Description ---------------------------------------------------------------- |------------------|---------------------- -[berkeley-db@4](https://formulae.brew.sh/formula/berkeley-db@4) | Berkeley DB | Wallet storage (only needed when wallet enabled) -[qt@5](https://formulae.brew.sh/formula/qt@5) | GUI | GUI toolkit (only needed when GUI enabled) -[qrencode](https://formulae.brew.sh/formula/qrencode) | QR codes in GUI | Generating QR codes (only needed when GUI enabled) -[zeromq](https://formulae.brew.sh/formula/zeromq) | ZMQ notification | Allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) -[sqlite](https://formulae.brew.sh/formula/sqlite) | SQLite DB | Wallet storage (only needed when wallet enabled) -[miniupnpc](https://formulae.brew.sh/formula/miniupnpc) | UPnP Support | Firewall-jumping support (needed for port mapping support) -[libnatpmp](https://formulae.brew.sh/formula/libnatpmp) | NAT-PMP Support | Firewall-jumping support (needed for port mapping support) -[python3](https://formulae.brew.sh/formula/python@3.9) | Testing | Python Interpreter (only needed when running the test suite) - -The following dependencies are **optional** packages required for deploying: - -Library | Purpose | Description -----------------------------------------------------|------------------|---------------------- -[librsvg](https://formulae.brew.sh/formula/librsvg) | Deploy Dependency| Library to render SVG files -[ds_store](https://pypi.org/project/ds-store/) | Deploy Dependency| Examine and modify .DS_Store files -[mac_alias](https://pypi.org/project/mac-alias/) | Deploy Dependency| Generate/Read binary alias and bookmark records - -See [dependencies.md](dependencies.md) for a complete overview. - ## Preparation The commands in this guide should be executed in a Terminal application. @@ -79,6 +42,9 @@ Note: If you run into issues while installing Homebrew or pulling packages, refe The first step is to download the required dependencies. These dependencies represent the packages required to get a barebones installation up and running. + +See [dependencies.md](dependencies.md) for a complete overview. + To install, run the following from your terminal: ``` bash @@ -100,29 +66,21 @@ git clone https://github.com/bitcoin/bitcoin.git #### Wallet Dependencies It is not necessary to build wallet functionality to run `bitcoind` or `bitcoin-qt`. -To enable legacy wallets, you must install `berkeley-db@4`. -To enable [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md), `sqlite` is required. -Skip `berkeley-db@4` if you intend to *exclusively* use descriptor wallets. - -###### Legacy Wallet Support -`berkeley-db@4` is required to enable support for legacy wallets. -Skip if you don't intend to use legacy wallets. +###### Descriptor Wallet Support -``` bash -brew install berkeley-db@4 -``` +`sqlite` is required to support for descriptor wallets. -###### Descriptor Wallet Support +macOS ships with a useable `sqlite` package, meaning you don't need to +install anything. -Note: Apple has included a useable `sqlite` package since macOS 10.14. -You may not need to install this package. +###### Legacy Wallet Support -`sqlite` is required to enable support for descriptor wallets. -Skip if you don't intend to use descriptor wallets. +`berkeley-db@4` is only required to support for legacy wallets. +Skip if you don't intend to use legacy wallets. ``` bash -brew install sqlite +brew install berkeley-db@4 ``` --- @@ -138,14 +96,6 @@ Skip if you don't intend to use the GUI. brew install qt@5 ``` -Ensure that the `qt@5` package is installed, not the `qt` package. -If 'qt' is installed, the build process will fail. -if installed, remove the `qt` package with the following command: - -``` bash -brew uninstall qt -``` - Note: Building with Qt binaries downloaded from the Qt website is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714). @@ -218,10 +168,6 @@ This command depends on a couple of python packages, so it is required that you Ensuring that `python` is installed, you can install the deploy dependencies by running the following commands in your terminal: -``` bash -brew install librsvg -``` - ``` bash pip3 install ds_store mac_alias ``` diff --git a/doc/build-unix.md b/doc/build-unix.md index 73c0bf877950..874015707a25 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -26,30 +26,7 @@ make install # optional This will build bitcoin-qt as well, if the dependencies are met. -Dependencies ---------------------- - -These dependencies are required: - - Library | Purpose | Description - ------------|------------------|---------------------- - libboost | Utility | Library for threading, data structures, etc - libevent | Networking | OS independent asynchronous networking - -Optional dependencies: - - Library | Purpose | Description - ------------|------------------|---------------------- - miniupnpc | UPnP Support | Firewall-jumping support - libnatpmp | NAT-PMP Support | Firewall-jumping support - libdb4.8 | Berkeley DB | Optional, wallet storage (only needed when wallet enabled) - qt | GUI | GUI toolkit (only needed when GUI enabled) - libqrencode | QR codes in GUI | Optional for generating QR codes (only needed when GUI enabled) - univalue | Utility | JSON parsing and encoding (bundled version will be used unless --with-system-univalue passed to configure) - libzmq3 | ZMQ notification | Optional, allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) - sqlite3 | SQLite DB | Optional, wallet storage (only needed when wallet enabled) - -For the versions used, see [dependencies.md](dependencies.md) +See [dependencies.md](dependencies.md) for a complete overview. Memory Requirements -------------------- @@ -82,21 +59,17 @@ Build requirements: Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: - sudo apt-get install libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev - -Berkeley DB is required for the wallet. - -Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, but these will install -Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed executables, which -are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, -pass `--with-incompatible-bdb` to configure. - -Otherwise, you can build Berkeley DB [yourself](#berkeley-db). + sudo apt-get install libevent-dev libboost-dev SQLite is required for the descriptor wallet: sudo apt install libsqlite3-dev +Berkeley DB is required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, +but these will install Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed +executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass +`--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). + To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) Optional port mapping libraries (see: `--with-miniupnpc`, `--enable-upnp-default`, and `--with-natpmp`, `--enable-natpmp-default`): @@ -107,6 +80,10 @@ ZMQ dependencies (provides ZMQ API): sudo apt-get install libzmq3-dev +User-Space, Statically Defined Tracing (USDT) dependencies: + + sudo apt install systemtap-sdt-dev + GUI dependencies: If you want to build bitcoin-qt, make sure that the required packages for Qt development @@ -117,6 +94,10 @@ To build with Qt 5 you need the following: sudo apt-get install libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools +Additionally, to support Wayland protocol for modern desktop environments: + + sudo apt install qtwayland5 + libqrencode (optional) can be installed with: sudo apt-get install libqrencode-dev @@ -137,20 +118,18 @@ Now, you can either build from self-compiled [depends](/depends/README.md) or in sudo dnf install libevent-devel boost-devel -Berkeley DB is required for the wallet: +SQLite is required for the descriptor wallet: + + sudo dnf install sqlite-devel + +Berkeley DB is required for the legacy wallet: sudo dnf install libdb4-devel libdb4-cxx-devel Newer Fedora releases, since Fedora 33, have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on Berkeley DB 4.8. If you do not care about wallet compatibility, -pass `--with-incompatible-bdb` to configure. - -Otherwise, you can build Berkeley DB [yourself](#berkeley-db). - -SQLite is required for the descriptor wallet: - - sudo dnf install sqlite-devel +pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) @@ -162,6 +141,10 @@ ZMQ dependencies (provides ZMQ API): sudo dnf install zeromq-devel +User-Space, Statically Defined Tracing (USDT) dependencies: + + sudo dnf install systemtap + GUI dependencies: If you want to build bitcoin-qt, make sure that the required packages for Qt development @@ -172,6 +155,10 @@ To build with Qt 5 you need the following: sudo dnf install qt5-qttools-devel qt5-qtbase-devel +Additionally, to support Wayland protocol for modern desktop environments: + + sudo dnf install qt5-qtwayland + libqrencode (optional) can be installed with: sudo dnf install qrencode-devel @@ -208,8 +195,10 @@ turned off by default. See the configure options for NAT-PMP behavior desired: Berkeley DB ----------- -It is recommended to use Berkeley DB 4.8. If you have to build it yourself, -you can use [the installation script included in contrib/](/contrib/install_db4.sh) + +The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is +recommended to use Berkeley DB 4.8. If you have to build it yourself, you can +use [the installation script included in contrib/](/contrib/install_db4.sh) like so: ```shell @@ -220,16 +209,7 @@ from the root of the repository. Otherwise, you can build Bitcoin Core from self-compiled [depends](/depends/README.md). -**Note**: You only need Berkeley DB if the wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). - -Boost ------ -If you need to build Boost yourself: - - sudo su - ./bootstrap.sh - ./bjam install - +**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). Security -------- @@ -279,12 +259,12 @@ Hardening enables the following features: Disable-wallet mode -------------------- -When the intention is to run only a P2P node without a wallet, Bitcoin Core may be compiled in -disable-wallet mode with: +When the intention is to only run a P2P node, without a wallet, Bitcoin Core can +be compiled in disable-wallet mode with: ./configure --disable-wallet -In this case there is no dependency on Berkeley DB 4.8 and SQLite. +In this case there is no dependency on SQLite or Berkeley DB. Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. @@ -297,42 +277,14 @@ A list of additional configure flags can be displayed with: Setup and Build Example: Arch Linux ----------------------------------- -This example lists the steps necessary to setup and build a command line only, non-wallet distribution of the latest changes on Arch Linux: +This example lists the steps necessary to setup and build a command line only distribution of the latest changes on Arch Linux: - pacman -S git base-devel boost libevent python + pacman --sync --needed autoconf automake boost gcc git libevent libtool make pkgconf python sqlite git clone https://github.com/bitcoin/bitcoin.git cd bitcoin/ ./autogen.sh - ./configure --disable-wallet --without-gui --without-miniupnpc + ./configure make check + ./src/bitcoind -Note: -Enabling wallet support requires either compiling against a Berkeley DB newer than 4.8 (package `db`) using `--with-incompatible-bdb`, -or building and depending on a local version of Berkeley DB 4.8. The readily available Arch Linux packages are currently built using -`--with-incompatible-bdb` according to the [PKGBUILD](https://projects.archlinux.org/svntogit/community.git/tree/bitcoin/trunk/PKGBUILD). -As mentioned above, when maintaining portability of the wallet between the standard Bitcoin Core distributions and independently built -node software is desired, Berkeley DB 4.8 must be used. - - -ARM Cross-compilation -------------------- -These steps can be performed on, for example, an Ubuntu VM. The depends system -will also work on other Linux distributions, however the commands for -installing the toolchain will be different. - -Make sure you install the build requirements mentioned above. -Then, install the toolchain and curl: - - sudo apt-get install g++-arm-linux-gnueabihf curl - -To build executables for ARM: - - cd depends - make HOST=arm-linux-gnueabihf NO_QT=1 - cd .. - ./autogen.sh - CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++ - make - - -For further documentation on the depends system see [README.md](../depends/README.md) in the depends directory. +If you intend to work with legacy Berkeley DB wallets, see [Berkeley DB](#berkeley-db) section. diff --git a/doc/build-windows.md b/doc/build-windows.md index f88b9739deb8..e35d3bcbd09a 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -5,11 +5,9 @@ Below are some notes on how to build Bitcoin Core for Windows. The options known to work for building Bitcoin Core on Windows are: -* On Linux, using the [Mingw-w64](https://mingw-w64.org/doku.php) cross compiler tool chain. Ubuntu Bionic 18.04 is required -and is the platform used to build the Bitcoin Core Windows release binaries. -* On Windows, using [Windows -Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and the Mingw-w64 cross compiler tool chain. -* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). +* On Linux, using the [Mingw-w64](https://www.mingw-w64.org/) cross compiler tool chain. +* On Windows, using [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and Mingw-w64. +* On Windows, using [Microsoft Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). Other options which may work, but which have not been extensively tested are (please contribute instructions): @@ -18,40 +16,12 @@ Other options which may work, but which have not been extensively tested are (pl Installing Windows Subsystem for Linux --------------------------------------- -With Windows 10, Microsoft has released a new feature named the [Windows -Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about). This -feature allows you to run a bash shell directly on Windows in an Ubuntu-based -environment. Within this environment you can cross compile for Windows without -the need for a separate Linux VM or server. Note that while WSL can be installed with -other Linux variants, such as OpenSUSE, the following instructions have only been -tested with Ubuntu. - -This feature is not supported in versions of Windows prior to Windows 10 or on -Windows Server SKUs. In addition, it is available [only for 64-bit versions of -Windows](https://docs.microsoft.com/windows/wsl/install-win10). - -Full instructions to install WSL are available on the above link. -To install WSL on Windows 10 with Fall Creators Update installed (version >= 16215.0) do the following: - -1. Enable the Windows Subsystem for Linux feature - * Open the Windows Features dialog (`OptionalFeatures.exe`) - * Enable 'Windows Subsystem for Linux' - * Click 'OK' and restart if necessary -2. Install Ubuntu - * Open Microsoft Store and search for "Ubuntu 18.04" or use [this link](https://www.microsoft.com/store/productId/9N9TNGVNDL3Q) - * Click Install -3. Complete Installation - * Open a cmd prompt and type "Ubuntu1804" - * Create a new UNIX user account (this is a separate account from your Windows account) - -After the bash shell is active, you can follow the instructions below, starting -with the "Cross-compilation" section. Compiling the 64-bit version is -recommended, but it is possible to compile the 32-bit version. +Follow the upstream installation instructions, available [here](https://docs.microsoft.com/windows/wsl/install-win10). Cross-compilation for Ubuntu and Windows Subsystem for Linux ------------------------------------------------------------ -The steps below can be performed on Ubuntu (including in a VM) or WSL. The depends system +The steps below can be performed on Ubuntu or WSL. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. @@ -78,28 +48,16 @@ Acquire the source in the usual way: ## Building for 64-bit Windows The first step is to install the mingw-w64 cross-compilation tool chain: + - on modern systems (Ubuntu 21.04 Hirsute Hippo or newer, Debian 11 Bullseye or newer): - sudo apt install g++-mingw-w64-x86-64 - -Next, set the default `mingw32 g++` compiler option to POSIX[1](#footnote1): - -``` -sudo update-alternatives --config x86_64-w64-mingw32-g++ +```sh +sudo apt install g++-mingw-w64-x86-64-posix ``` -After running the above command, you should see output similar to that below. -Choose the option that ends with `posix`. + - on older systems: -``` -There are 2 choices for the alternative x86_64-w64-mingw32-g++ (providing /usr/bin/x86_64-w64-mingw32-g++). - - Selection Path Priority Status ------------------------------------------------------------- - 0 /usr/bin/x86_64-w64-mingw32-g++-win32 60 auto mode -* 1 /usr/bin/x86_64-w64-mingw32-g++-posix 30 manual mode - 2 /usr/bin/x86_64-w64-mingw32-g++-win32 60 manual mode - -Press to keep the current choice[*], or type selection number: +```sh +sudo apt install g++-mingw-w64-x86-64 ``` Once the toolchain is installed the build steps are common: @@ -142,13 +100,3 @@ way. This will install to `c:\workspace\bitcoin`, for example: You can also create an installer using: make deploy - -Footnotes ---------- - -1: Starting from Ubuntu Xenial 16.04, both the 32 and 64 bit Mingw-w64 packages install two different -compiler options to allow a choice between either posix or win32 threads. The default option is win32 threads which is the more -efficient since it will result in binary code that links directly with the Windows kernel32.lib. Unfortunately, the headers -required to support win32 threads conflict with some of the classes in the C++11 standard library, in particular std::mutex. -It's not possible to build the Bitcoin Core code using the win32 version of the Mingw-w64 cross compilers (at least not without -modifying headers in the Bitcoin Core source code). diff --git a/doc/cjdns.md b/doc/cjdns.md new file mode 100644 index 000000000000..b69564729f86 --- /dev/null +++ b/doc/cjdns.md @@ -0,0 +1,116 @@ +# CJDNS support in Bitcoin Core + +It is possible to run Bitcoin Core over CJDNS, an encrypted IPv6 network that +uses public-key cryptography for address allocation and a distributed hash table +for routing. + +## What is CJDNS? + +CJDNS is like a distributed, shared VPN with multiple entry points where every +participant can reach any other participant. All participants use addresses from +the `fc00::/8` network (reserved IPv6 range). Installation and configuration is +done outside of Bitcoin Core, similarly to a VPN (either in the host/OS or on +the network router). See https://github.com/cjdelisle/cjdns#readme and +https://github.com/hyperboria/docs#hyperboriadocs for more information. + +Compared to IPv4/IPv6, CJDNS provides end-to-end encryption and protects nodes +from traffic analysis and filtering. + +Used with Tor and I2P, CJDNS is a complementary option that can enhance network +redundancy and robustness for both the Bitcoin network and individual nodes. + +Each network has different characteristics. For instance, Tor is widely used but +somewhat centralized. I2P connections have a source address and I2P is slow. +CJDNS is fast but does not hide the sender and the recipient from intermediate +routers. + +## Installing CJDNS and finding a peer to connect to the network + +To install and set up CJDNS, follow the instructions at +https://github.com/cjdelisle/cjdns#how-to-install-cjdns. + +You need to initiate an outbound connection to a peer on the CJDNS network +before it will work with your Bitcoin Core node. This is described in steps +["2. Find a friend"](https://github.com/cjdelisle/cjdns#2-find-a-friend) and +["3. Connect your node to your friend's +node"](https://github.com/cjdelisle/cjdns#3-connect-your-node-to-your-friends-node) +in the CJDNS documentation. + +One quick way to accomplish these two steps is to query for available public +peers on [Hyperboria](https://github.com/hyperboria) by running the following: + +``` +git clone https://github.com/hyperboria/peers hyperboria-peers +cd hyperboria-peers +./testAvailable.py +``` + +For each peer, the `./testAvailable.py` script prints the filename of the peer's +credentials followed by the ping result. + +Choose one or several peers, copy their credentials from their respective files, +paste them into the relevant IPv4 or IPv6 "connectTo" JSON object in the +`cjdroute.conf` file you created in step ["1. Generate a new configuration +file"](https://github.com/cjdelisle/cjdns#1-generate-a-new-configuration-file), +and save the file. + +## Launching CJDNS + +Typically, CJDNS might be launched from its directory with +`sudo ./cjdroute < cjdroute.conf` and it sheds permissions after setting up the +[TUN](https://en.wikipedia.org/wiki/TUN/TAP) interface. You may also [launch it as an +unprivileged user](https://github.com/cjdelisle/cjdns/blob/master/doc/non-root-user.md) +with some additional setup. + +The network connection can be checked by running `./tools/peerStats` from the +CJDNS directory. + +## Run Bitcoin Core with CJDNS + +Once you are connected to the CJDNS network, the following Bitcoin Core +configuration option makes CJDNS peers automatically reachable: + +``` +-cjdnsreachable +``` + +When enabled, this option tells Bitcoin Core that it is running in an +environment where a connection to an `fc00::/8` address will be to the CJDNS +network instead of to an [RFC4193](https://datatracker.ietf.org/doc/html/rfc4193) +IPv6 local network. This helps Bitcoin Core perform better address management: + - Your node can consider incoming `fc00::/8` connections to be from the CJDNS + network rather than from an IPv6 private one. + - If one of your node's local addresses is `fc00::/8`, then it can choose to + gossip that address to peers. + +## Additional configuration options related to CJDNS + +``` +-onlynet=cjdns +``` + +Make automatic outbound connections only to CJDNS addresses. Inbound and manual +connections are not affected by this option. It can be specified multiple times +to allow multiple networks, e.g. onlynet=cjdns, onlynet=i2p, onlynet=onion. + +CJDNS support was added to Bitcoin Core in version 23.0 and there may be fewer +CJDNS peers than Tor or IP ones. You can use `bitcoin-cli -addrinfo` to see the +number of CJDNS addresses known to your node. + +In general, a node can be run with both an onion service and CJDNS (or any/all +of IPv4/IPv6/onion/I2P/CJDNS), which can provide a potential fallback if one of +the networks has issues. There are a number of ways to configure this; see +[doc/tor.md](https://github.com/bitcoin/bitcoin/blob/master/doc/tor.md) for +details. + +## CJDNS-related information in Bitcoin Core + +There are several ways to see your CJDNS address in Bitcoin Core: +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` + +To see which CJDNS peers your node is connected to, use `bitcoin-cli -netinfo 4` +or the `getpeerinfo` RPC (i.e. `bitcoin-cli getpeerinfo`). + +To see which CJDNS addresses your node knows, use the `getnodeaddresses 0 cjdns` +RPC. diff --git a/doc/dependencies.md b/doc/dependencies.md index 66c5a76b3b8b..ef8faff06c41 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -1,48 +1,50 @@ -Dependencies -============ - -These are the dependencies currently used by Bitcoin Core. You can find instructions for installing them in the `build-*.md` file for your platform. - -| Dependency | Version used | Minimum required | CVEs | Shared | [Bundled Qt library](https://doc.qt.io/qt-5/configure-options.html#third-party-libraries) | -| --- | --- | --- | --- | --- | --- | -| Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | -| Boost | [1.71.0](https://www.boost.org/users/download/) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | | -| Clang[ \* ](#note1) | | [5.0+](https://releases.llvm.org/download.html) (C++17 support) | | | | -| Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | -| fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | -| FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | -| GCC | | [7+](https://gcc.gnu.org/) (C++17 support) | | | | -| HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| libevent | [2.1.12-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | -| libnatpmp | git commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | | | -| libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| librsvg | | | | | | -| MiniUPnPc | [2.2.2](https://miniupnp.tuxfamily.org/files) | | No | | | -| PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| Python (tests) | | [3.6](https://www.python.org/downloads) | | | | -| qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.12.11](https://download.qt.io/official_releases/qt/) | [5.9.5](https://github.com/bitcoin/bitcoin/issues/20104) | No | | | -| SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | -| XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | -| xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | -| ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | -| zlib | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | - -Note \* : When compiling with `-stdlib=libc++`, the minimum supported libc++ version is 7.0. - -Controlling dependencies ------------------------- -Some dependencies are not needed in all configurations. The following are some factors that affect the dependency list. - -#### Options passed to `./configure` -* MiniUPnPc is not needed with `--without-miniupnpc`. -* libnatpmp is not needed with `--without-natpmp`. -* Berkeley DB is not needed with `--disable-wallet` or `--without-bdb`. -* SQLite is not needed with `--disable-wallet` or `--without-sqlite`. -* Qt is not needed with `--without-gui`. -* If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. -* ZeroMQ is needed only with the `--with-zmq` option. - -#### Other -* librsvg is only needed if you need to run `make deploy` on (cross-compilation to) macOS. -* Not-Qt-bundled zlib is required to build the [DMG tool](../contrib/macdeploy/README.md#deterministic-macos-dmg-notes) from the libdmg-hfsplus project. +# Dependencies + +These are the dependencies used by Bitcoin Core. +You can find installation instructions in the `build-*.md` file for your platform. +"Runtime" and "Version Used" are both in reference to the release binaries. + +| Dependency | Minimum required | +| --- | --- | +| [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | +| [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | +| [Clang](https://clang.llvm.org) | [8.0](https://github.com/bitcoin/bitcoin/pull/24164) | +| [GCC](https://gcc.gnu.org) | [8.1](https://github.com/bitcoin/bitcoin/pull/23060) | +| [Python](https://www.python.org) (tests) | [3.6](https://github.com/bitcoin/bitcoin/pull/19504) | +| [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | + +## Required + +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.80.0](https://github.com/bitcoin/bitcoin/pull/25873) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | +| [libevent](../depends/packages/libevent.mk) | [link](https://github.com/libevent/libevent/releases) | [2.1.12-stable](https://github.com/bitcoin/bitcoin/pull/21991) | [2.1.8](https://github.com/bitcoin/bitcoin/pull/24681) | No | +| glibc | [link](https://www.gnu.org/software/libc/) | N/A | [2.18](https://github.com/bitcoin/bitcoin/pull/23511) | Yes | +| Linux Kernel | [link](https://www.kernel.org/) | N/A | 3.2.0 | Yes | + +## Optional + +### GUI +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | +| [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | +| [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [3.4.4](https://github.com/bitcoin/bitcoin/pull/6373) | | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.5](https://github.com/bitcoin/bitcoin/pull/25719) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | + +### Networking +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [07004b9...](https://github.com/bitcoin/bitcoin/pull/25917) | | No | +| [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.2](https://github.com/bitcoin/bitcoin/pull/20421) | 1.9 | No | + +### Notifications +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [ZeroMQ](../depends/packages/zeromq.mk) | [link](https://github.com/zeromq/libzmq/releases) | [4.3.4](https://github.com/bitcoin/bitcoin/pull/23956) | 4.0.0 | No | + +### Wallet +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [Berkeley DB](../depends/packages/bdb.mk) (legacy wallet) | [link](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.30 | 4.8.x | No | +| [SQLite](../depends/packages/sqlite.mk) | [link](https://sqlite.org) | [3.32.1](https://github.com/bitcoin/bitcoin/pull/19077) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | No | diff --git a/doc/descriptors.md b/doc/descriptors.md index e27ff8754684..1baf652f3085 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -11,13 +11,18 @@ Supporting RPCs are: addresses. - `listunspent` outputs a specialized descriptor for the reported unspent outputs. - `getaddressinfo` outputs a descriptor for solvable addresses (since v0.18). -- `importmulti` takes as input descriptors to import into the wallet +- `importmulti` takes as input descriptors to import into a legacy wallet (since v0.18). - `generatetodescriptor` takes as input a descriptor and generates coins to it (`regtest` only, since v0.19). - `utxoupdatepsbt` takes as input descriptors to add information to the psbt (since v0.19). -- `createmultisig` and `addmultisigaddress` return descriptors as well (since v0.20) +- `createmultisig` and `addmultisigaddress` return descriptors as well (since v0.20). +- `importdescriptors` takes as input descriptors to import into a descriptor wallet + (since v0.21). +- `listdescriptors` outputs descriptors imported into a descriptor wallet (since v22). +- `scanblocks` takes as input descriptors to scan for in blocks and returns the + relevant blockhashes (since v25). This document describes the language. For the specifics on usage, see the RPC documentation for the functions mentioned above. @@ -33,6 +38,7 @@ Output descriptors currently support: - Pay-to-taproot outputs (P2TR), through the `tr` function. - Multisig scripts, through the `multi` function. - Multisig scripts where the public keys are sorted lexicographically, through the `sortedmulti` function. +- Multisig scripts inside taproot script trees, through the `multi_a` (and `sortedmulti_a`) function. - Any type of supported address through the `addr` function. - Raw hex scripts through the `raw` function. - Public keys (compressed and uncompressed) in hex notation, or BIP32 extended pubkeys with derivation paths. @@ -56,6 +62,7 @@ Output descriptors currently support: - `wsh(multi(1,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1/0/*,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/0/0/*))` describes a set of *1-of-2* P2WSH multisig outputs where the first multisig key is the *1/0/`i`* child of the first specified xpub and the second multisig key is the *0/0/`i`* child of the second specified xpub, and `i` is any number in a configurable range (`0-1000` by default). - `wsh(sortedmulti(1,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1/0/*,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/0/0/*))` describes a set of *1-of-2* P2WSH multisig outputs where one multisig key is the *1/0/`i`* child of the first specified xpub and the other multisig key is the *0/0/`i`* child of the second specified xpub, and `i` is any number in a configurable range (`0-1000` by default). The order of public keys in the resulting witnessScripts is determined by the lexicographic order of the public keys at that index. - `tr(c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5,{pk(fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),pk(e493dbf1c10d80f3581e4904930b1404cc6c13900ee0758474fa94abe8c4cd13)})` describes a P2TR output with the `c6...` x-only pubkey as internal key, and two script paths. +- `tr(c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5,sortedmulti_a(2,2f8bde4d1a07209355b4a7250a5c5128e88b84bddc619ab7cba8d569b240efe4,5cbdf0646e5db4eaa398f365f2ea7a0e3d419b7e0330e39ce92bddedcac4f9bc))` describes a P2TR output with the `c6...` x-only pubkey as internal key, and a single `multi_a` script that needs 2 signatures with 2 specified x-only keys, which will be sorted lexicographically. ## Reference @@ -68,11 +75,14 @@ Descriptors consist of several types of expressions. The top level expression is - `pkh(KEY)` (not inside `tr`): P2PKH output for the given public key (use `addr` if you only know the pubkey hash). - `wpkh(KEY)` (top level or inside `sh` only): P2WPKH output for the given compressed pubkey. - `combo(KEY)` (top level only): an alias for the collection of `pk(KEY)` and `pkh(KEY)`. If the key is compressed, it also includes `wpkh(KEY)` and `sh(wpkh(KEY))`. -- `multi(k,KEY_1,KEY_2,...,KEY_n)` (not inside `tr`): k-of-n multisig script. +- `multi(k,KEY_1,KEY_2,...,KEY_n)` (not inside `tr`): k-of-n multisig script using OP_CHECKMULTISIG. - `sortedmulti(k,KEY_1,KEY_2,...,KEY_n)` (not inside `tr`): k-of-n multisig script with keys sorted lexicographically in the resulting script. +- `multi_a(k,KEY_1,KEY_2,...,KEY_N)` (only inside `tr`): k-of-n multisig script using OP_CHECKSIG, OP_CHECKSIGADD, and OP_NUMEQUAL. +- `sortedmulti_a(k,KEY_1,KEY_2,...,KEY_N)` (only inside `tr`): similar to `multi_a`, but the (x-only) public keys in it will be sorted lexicographically. - `tr(KEY)` or `tr(KEY,TREE)` (top level only): P2TR output with the specified key as internal key, and optionally a tree of script paths. - `addr(ADDR)` (top level only): the script which ADDR expands to. - `raw(HEX)` (top level only): the script whose hex encoding is HEX. +- `rawtr(KEY)` (top level only): P2TR output with the specified key as output key. NOTE: while it's possible to use this to construct wallets, it has several downsides, like being unable to prove no hidden script path exists. Use at your own risk. `KEY` expressions: - Optionally, key origin information, consisting of: @@ -83,23 +93,23 @@ Descriptors consist of several types of expressions. The top level expression is - Followed by the actual key, which is either: - Hex encoded public keys (either 66 characters starting with `02` or `03` for a compressed pubkey, or 130 characters starting with `04` for an uncompressed pubkey). - Inside `wpkh` and `wsh`, only compressed public keys are permitted. - - Inside `tr`, x-only pubkeys are also permitted (64 hex characters). + - Inside `tr` and `rawtr`, x-only pubkeys are also permitted (64 hex characters). - [WIF](https://en.bitcoin.it/wiki/Wallet_import_format) encoded private keys may be specified instead of the corresponding public key, with the same meaning. - `xpub` encoded extended public key or `xprv` encoded extended private key (as defined in [BIP 32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)). - Followed by zero or more `/NUM` unhardened and `/NUM'` hardened BIP32 derivation steps. - Optionally followed by a single `/*` or `/*'` final step to denote all (direct) unhardened or hardened children. - The usage of hardened derivation steps requires providing the private key. +(Anywhere a `'` suffix is permitted to denote hardened derivation, the suffix `h` can be used instead.) + `TREE` expressions: - any `SCRIPT` expression - An open brace `{`, a `TREE` expression, a comma `,`, a `TREE` expression, and a closing brace `}` -(Anywhere a `'` suffix is permitted to denote hardened derivation, the suffix `h` can be used instead.) - `ADDR` expressions are any type of supported address: - P2PKH addresses (base58, of the form `1...` for mainnet or `[nm]...` for testnet). Note that P2PKH addresses in descriptors cannot be used for P2PK outputs (use the `pk` function instead). - P2SH addresses (base58, of the form `3...` for mainnet or `2...` for testnet, defined in [BIP 13](https://github.com/bitcoin/bips/blob/master/bip-0013.mediawiki)). -- Segwit addresses (bech32, of the form `bc1...` for mainnet or `tb1...` for testnet, defined in [BIP 173](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki)). +- Segwit addresses (bech32 and bech32m, of the form `bc1...` for mainnet or `tb1...` for testnet, defined in [BIP 173](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki) and [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki)). ## Explanation @@ -139,6 +149,47 @@ Key order does not matter for `sortedmulti()`. `sortedmulti()` behaves in the sa as `multi()` does but the keys are reordered in the resulting script such that they are lexicographically ordered as described in BIP67. +#### Basic multisig example + +For a good example of a basic M-of-N multisig between multiple participants using descriptor +wallets and PSBTs, as well as a signing flow, see [this functional test](/test/functional/wallet_multisig_descriptor_psbt.py). + +Disclaimers: It is important to note that this example serves as a quick-start and is kept basic for readability. A downside of the approach +outlined here is that each participant must maintain (and backup) two separate wallets: a signer and the corresponding multisig. +It should also be noted that privacy best-practices are not "by default" here - participants should take care to only use the signer to sign +transactions related to the multisig. Lastly, it is not recommended to use anything other than a Bitcoin Core descriptor wallet to serve as your +signer(s). Other wallets, whether hardware or software, likely impose additional checks and safeguards to prevent users from signing transactions that +could lead to loss of funds, or are deemed security hazards. Conforming to various 3rd-party checks and verifications is not in the scope of this example. + +The basic steps are: + + 1. Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet which we will refer to as + the participant's signer wallet. Avoid reusing this wallet for any purpose other than signing transactions from the + corresponding multisig we are about to create. Hint: extract the wallet's xpubs using `listdescriptors` and pick the one from the + `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses) + 2. Create a watch-only descriptor wallet (blank, private keys disabled). Now the multisig is created by importing the two descriptors: + `wsh(sortedmulti(,XPUB1/0/*,XPUB2/0/*,…,XPUBN/0/*))` and `wsh(sortedmulti(,XPUB1/1/*,XPUB2/1/*,…,XPUBN/1/*))` + (one descriptor w/ `0` for receiving addresses and another w/ `1` for change). Every participant does this + 3. A receiving address is generated for the multisig. As a check to ensure step 2 was done correctly, every participant + should verify they get the same addresses + 4. Funds are sent to the resulting address + 5. A sending transaction from the multisig is created using `walletcreatefundedpsbt` (anyone can initiate this). It is simple to do + this in the GUI by going to the `Send` tab in the multisig wallet and creating an unsigned transaction (PSBT) + 6. At least `M` participants check the PSBT with their multisig using `decodepsbt` to verify the transaction is OK before signing it. + 7. (If OK) the participant signs the PSBT with their signer wallet using `walletprocesspsbt`. It is simple to do this in the GUI by + loading the PSBT from file and signing it + 8. The signed PSBTs are collected with `combinepsbt`, finalized w/ `finalizepsbt`, and then the resulting transaction is broadcasted + to the network. Note that any wallet (eg one of the signers or multisig) is capable of doing this. + 9. Checks that balances are correct after the transaction has been included in a block + +You may prefer a daisy chained signing flow where each participant signs the PSBT one after another until +the PSBT has been signed `M` times and is "complete." For the most part, the steps above remain the same, except (6, 7) +change slightly from signing the original PSBT in parallel to signing it in series. `combinepsbt` is not necessary with +this signing flow and the last (`m`th) signer can just broadcast the PSBT after signing. Note that a parallel signing flow may be +preferable in cases where there are more signers. This signing flow is also included in the test / Python example. +[The test](/test/functional/wallet_multisig_descriptor_psbt.py) is meant to be documentation as much as it is a functional test, so +it is kept as simple and readable as possible. + ### BIP32 derived keys and chains Most modern wallet software and hardware uses keys that are derived using @@ -215,6 +266,6 @@ ones. For larger numbers of errors, or other types of errors, there is a roughly 1 in a trillion chance of not detecting the errors. All RPCs in Bitcoin Core will include the checksum in their output. Only -certain RPCs require checksums on input, including `deriveaddress` and +certain RPCs require checksums on input, including `deriveaddresses` and `importmulti`. The checksum for a descriptor without one can be computed using the `getdescriptorinfo` RPC. diff --git a/doc/design/assumeutxo.md b/doc/design/assumeutxo.md new file mode 100644 index 000000000000..ea51b1b87f6b --- /dev/null +++ b/doc/design/assumeutxo.md @@ -0,0 +1,139 @@ +# assumeutxo + +Assumeutxo is a feature that allows fast bootstrapping of a validating bitcoind +instance with a very similar security model to assumevalid. + +The RPC commands `dumptxoutset` and `loadtxoutset` are used to respectively generate +and load UTXO snapshots. The utility script `./contrib/devtools/utxo_snapshot.sh` may +be of use. + +## General background + +- [assumeutxo proposal](https://github.com/jamesob/assumeutxo-docs/tree/2019-04-proposal/proposal) +- [Github issue](https://github.com/bitcoin/bitcoin/issues/15605) +- [draft PR](https://github.com/bitcoin/bitcoin/pull/15606) + +## Design notes + +- A new block index `nStatus` flag is introduced, `BLOCK_ASSUMED_VALID`, to mark block + index entries that are required to be assumed-valid by a chainstate created + from a UTXO snapshot. This flag is mostly used as a way to modify certain + CheckBlockIndex() logic to account for index entries that are pending validation by a + chainstate running asynchronously in the background. We also use this flag to control + which index entries are added to setBlockIndexCandidates during LoadBlockIndex(). + +- Indexing implementations via BaseIndex can no longer assume that indexation happens + sequentially, since background validation chainstates can submit BlockConnected + events out of order with the active chain. + +- The concept of UTXO snapshots is treated as an implementation detail that lives + behind the ChainstateManager interface. The external presentation of the changes + required to facilitate the use of UTXO snapshots is the understanding that there are + now certain regions of the chain that can be temporarily assumed to be valid (using + the nStatus flag mentioned above). In certain cases, e.g. wallet rescanning, this is + very similar to dealing with a pruned chain. + + Logic outside ChainstateManager should try not to know about snapshots, instead + preferring to work in terms of more general states like assumed-valid. + + +## Chainstate phases + +Chainstate within the system goes through a number of phases when UTXO snapshots are +used, as managed by `ChainstateManager`. At various points there can be multiple +`Chainstate` objects in existence to facilitate both maintaining the network tip and +performing historical validation of the assumed-valid chain. + +It is worth noting that though there are multiple separate chainstates, those +chainstates share use of a common block index (i.e. they hold the same `BlockManager` +reference). + +The subheadings below outline the phases and the corresponding changes to chainstate +data. + +### "Normal" operation via initial block download + +`ChainstateManager` manages a single Chainstate object, for which +`m_snapshot_blockhash` is null. This chainstate is (maybe obviously) +considered active. This is the "traditional" mode of operation for bitcoind. + +| | | +| ---------- | ----------- | +| number of chainstates | 1 | +| active chainstate | ibd | + +### User loads a UTXO snapshot via `loadtxoutset` RPC + +`ChainstateManager` initializes a new chainstate (see `ActivateSnapshot()`) to load the +snapshot contents into. During snapshot load and validation (see +`PopulateAndValidateSnapshot()`), the new chainstate is not considered active and the +original chainstate remains in use as active. + +| | | +| ---------- | ----------- | +| number of chainstates | 2 | +| active chainstate | ibd | + +Once the snapshot chainstate is loaded and validated, it is promoted to active +chainstate and a sync to tip begins. A new chainstate directory is created in the +datadir for the snapshot chainstate called `chainstate_snapshot`. When this directory +is present in the datadir, the snapshot chainstate will be detected and loaded as +active on node startup (via `DetectSnapshotChainstate()`). + +| | | +| ---------- | ----------- | +| number of chainstates | 2 | +| active chainstate | snapshot | + +The snapshot begins to sync to tip from its base block, technically in parallel with +the original chainstate, but it is given priority during block download and is +allocated most of the cache (see `MaybeRebalanceCaches()` and usages) as our chief +consideration is getting to network tip. + +**Failure consideration:** if shutdown happens at any point during this phase, both +chainstates will be detected during the next init and the process will resume. + +### Snapshot chainstate hits network tip + +Once the snapshot chainstate leaves IBD, caches are rebalanced +(via `MaybeRebalanceCaches()` in `ActivateBestChain()`) and more cache is given +to the background chainstate, which is responsible for doing full validation of the +assumed-valid parts of the chain. + +**Note:** at this point, ValidationInterface callbacks will be coming in from both +chainstates. Considerations here must be made for indexing, which may no longer be happening +sequentially. + +### Background chainstate hits snapshot base block + +Once the tip of the background chainstate hits the base block of the snapshot +chainstate, we stop use of the background chainstate by setting `m_stop_use` (not yet +committed - see #15606), in `CompleteSnapshotValidation()`, which is checked in +`ActivateBestChain()`). We hash the background chainstate's UTXO set contents and +ensure it matches the compiled value in `CMainParams::m_assumeutxo_data`. + +The background chainstate data lingers on disk until shutdown, when in +`ChainstateManager::Reset()`, the background chainstate is cleaned up with +`ValidatedSnapshotShutdownCleanup()`, which renames the `chainstate_[hash]` datadir as +`chainstate`. + +| | | +| ---------- | ----------- | +| number of chainstates | 2 (ibd has `m_stop_use=true`) | +| active chainstate | snapshot | + +**Failure consideration:** if bitcoind unexpectedly halts after `m_stop_use` is set on +the background chainstate but before `CompleteSnapshotValidation()` can finish, the +need to complete snapshot validation will be detected on subsequent init by +`ChainstateManager::CheckForUncleanShutdown()`. + +### Bitcoind restarts sometime after snapshot validation has completed + +When bitcoind initializes again, what began as the snapshot chainstate is now +indistinguishable from a chainstate that has been built from the traditional IBD +process, and will be initialized as such. + +| | | +| ---------- | ----------- | +| number of chainstates | 1 | +| active chainstate | ibd | diff --git a/doc/design/libraries.md b/doc/design/libraries.md new file mode 100644 index 000000000000..7cda64e713a0 --- /dev/null +++ b/doc/design/libraries.md @@ -0,0 +1,104 @@ +# Libraries + +| Name | Description | +|--------------------------|-------------| +| *libbitcoin_cli* | RPC client functionality used by *bitcoin-cli* executable | +| *libbitcoin_common* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_util*, but higher-level (see [Dependencies](#dependencies)). | +| *libbitcoin_consensus* | Stable, backwards-compatible consensus functionality used by *libbitcoin_node* and *libbitcoin_wallet* and also exposed as a [shared library](../shared-libraries.md). | +| *libbitcoinconsensus* | Shared library build of static *libbitcoin_consensus* library | +| *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node* and also exposed as a [shared library](../shared-libraries.md). | +| *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables | +| *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`--enable-multiprocess`](multiprocess.md) is used. | +| *libbitcoin_node* | P2P and RPC server functionality used by *bitcoind* and *bitcoin-qt* executables. | +| *libbitcoin_util* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_common*, but lower-level (see [Dependencies](#dependencies)). | +| *libbitcoin_wallet* | Wallet functionality used by *bitcoind* and *bitcoin-wallet* executables. | +| *libbitcoin_wallet_tool* | Lower-level wallet functionality used by *bitcoin-wallet* executable. | +| *libbitcoin_zmq* | [ZeroMQ](../zmq.md) functionality used by *bitcoind* and *bitcoin-qt* executables. | + +## Conventions + +- Most libraries are internal libraries and have APIs which are completely unstable! There are few or no restrictions on backwards compatibility or rules about external dependencies. Exceptions are *libbitcoin_consensus* and *libbitcoin_kernel* which have external interfaces documented at [../shared-libraries.md](../shared-libraries.md). + +- Generally each library should have a corresponding source directory and namespace. Source code organization is a work in progress, so it is true that some namespaces are applied inconsistently, and if you look at [`libbitcoin_*_SOURCES`](../../src/Makefile.am) lists you can see that many libraries pull in files from outside their source directory. But when working with libraries, it is good to follow a consistent pattern like: + + - *libbitcoin_node* code lives in `src/node/` in the `node::` namespace + - *libbitcoin_wallet* code lives in `src/wallet/` in the `wallet::` namespace + - *libbitcoin_ipc* code lives in `src/ipc/` in the `ipc::` namespace + - *libbitcoin_util* code lives in `src/util/` in the `util::` namespace + - *libbitcoin_consensus* code lives in `src/consensus/` in the `Consensus::` namespace + +## Dependencies + +- Libraries should minimize what other libraries they depend on, and only reference symbols following the arrows shown in the dependency graph below: + +
+ +```mermaid + +%%{ init : { "flowchart" : { "curve" : "basis" }}}%% + +graph TD; + +bitcoin-cli[bitcoin-cli]-->libbitcoin_cli; + +bitcoind[bitcoind]-->libbitcoin_node; +bitcoind[bitcoind]-->libbitcoin_wallet; + +bitcoin-qt[bitcoin-qt]-->libbitcoin_node; +bitcoin-qt[bitcoin-qt]-->libbitcoinqt; +bitcoin-qt[bitcoin-qt]-->libbitcoin_wallet; + +bitcoin-wallet[bitcoin-wallet]-->libbitcoin_wallet; +bitcoin-wallet[bitcoin-wallet]-->libbitcoin_wallet_tool; + +libbitcoin_cli-->libbitcoin_util; +libbitcoin_cli-->libbitcoin_common; + +libbitcoin_common-->libbitcoin_consensus; +libbitcoin_common-->libbitcoin_util; + +libbitcoin_kernel-->libbitcoin_consensus; +libbitcoin_kernel-->libbitcoin_util; + +libbitcoin_node-->libbitcoin_consensus; +libbitcoin_node-->libbitcoin_kernel; +libbitcoin_node-->libbitcoin_common; +libbitcoin_node-->libbitcoin_util; + +libbitcoinqt-->libbitcoin_common; +libbitcoinqt-->libbitcoin_util; + +libbitcoin_wallet-->libbitcoin_common; +libbitcoin_wallet-->libbitcoin_util; + +libbitcoin_wallet_tool-->libbitcoin_wallet; +libbitcoin_wallet_tool-->libbitcoin_util; + +classDef bold stroke-width:2px, font-weight:bold, font-size: smaller; +class bitcoin-qt,bitcoind,bitcoin-cli,bitcoin-wallet bold +``` +
+ +**Dependency graph**. Arrows show linker symbol dependencies. *Consensus* lib depends on nothing. *Util* lib is depended on by everything. *Kernel* lib depends only on consensus and util. + +
+ +- The graph shows what _linker symbols_ (functions and variables) from each library other libraries can call and reference directly, but it is not a call graph. For example, there is no arrow connecting *libbitcoin_wallet* and *libbitcoin_node* libraries, because these libraries are intended to be modular and not depend on each other's internal implementation details. But wallet code is still able to call node code indirectly through the `interfaces::Chain` abstract class in [`interfaces/chain.h`](../../src/interfaces/chain.h) and node code calls wallet code through the `interfaces::ChainClient` and `interfaces::Chain::Notifications` abstract classes in the same file. In general, defining abstract classes in [`src/interfaces/`](../../src/interfaces/) can be a convenient way of avoiding unwanted direct dependencies or circular dependencies between libraries. + +- *libbitcoin_consensus* should be a standalone dependency that any library can depend on, and it should not depend on any other libraries itself. + +- *libbitcoin_util* should also be a standalone dependency that any library can depend on, and it should not depend on other internal libraries. + +- *libbitcoin_common* should serve a similar function as *libbitcoin_util* and be a place for miscellaneous code used by various daemon, GUI, and CLI applications and libraries to live. It should not depend on anything other than *libbitcoin_util* and *libbitcoin_consensus*. The boundary between _util_ and _common_ is a little fuzzy but historically _util_ has been used for more generic, lower-level things like parsing hex, and _common_ has been used for bitcoin-specific, higher-level things like parsing base58. The difference between util and common is mostly important because *libbitcoin_kernel* is not supposed to depend on *libbitcoin_common*, only *libbitcoin_util*. In general, if it is ever unclear whether it is better to add code to *util* or *common*, it is probably better to add it to *common* unless it is very generically useful or useful particularly to include in the kernel. + + +- *libbitcoin_kernel* should only depend on *libbitcoin_util* and *libbitcoin_consensus*. + +- The only thing that should depend on *libbitcoin_kernel* internally should be *libbitcoin_node*. GUI and wallet libraries *libbitcoinqt* and *libbitcoin_wallet* in particular should not depend on *libbitcoin_kernel* and the unneeded functionality it would pull in, like block validation. To the extent that GUI and wallet code need scripting and signing functionality, they should be get able it from *libbitcoin_consensus*, *libbitcoin_common*, and *libbitcoin_util*, instead of *libbitcoin_kernel*. + +- GUI, node, and wallet code internal implementations should all be independent of each other, and the *libbitcoinqt*, *libbitcoin_node*, *libbitcoin_wallet* libraries should never reference each other's symbols. They should only call each other through [`src/interfaces/`](`../../src/interfaces/`) abstract interfaces. + +## Work in progress + +- Validation code is moving from *libbitcoin_node* to *libbitcoin_kernel* as part of [The libbitcoinkernel Project #24303](https://github.com/bitcoin/bitcoin/issues/24303) +- Source code organization is discussed in general in [Library source code organization #15732](https://github.com/bitcoin/bitcoin/issues/15732) diff --git a/doc/multiprocess.md b/doc/design/multiprocess.md similarity index 100% rename from doc/multiprocess.md rename to doc/design/multiprocess.md diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 583c50a76363..00c68911efac 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -12,10 +12,12 @@ Developer Notes - [Generating Documentation](#generating-documentation) - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) + - [Show sources in debugging](#show-sources-in-debugging) - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - [`debug.log`](#debuglog) - [Signet, testnet, and regtest modes](#signet-testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) + - [DEBUG_LOCKCONTENTION](#debug_lockcontention) - [Valgrind suppressions file](#valgrind-suppressions-file) - [Compiling for test coverage](#compiling-for-test-coverage) - [Performance profiling with perf](#performance-profiling-with-perf) @@ -30,6 +32,7 @@ Developer Notes - [C++ data structures](#c-data-structures) - [Strings and formatting](#strings-and-formatting) - [Shadowing](#shadowing) + - [Lifetimebound](#lifetimebound) - [Threads and synchronization](#threads-and-synchronization) - [Scripts](#scripts) - [Shebang](#shebang) @@ -89,8 +92,15 @@ code. - Class member variables have a `m_` prefix. - Global variables have a `g_` prefix. - Constant names are all uppercase, and use `_` to separate words. + - Enumerator constants may be `snake_case`, `PascalCase` or `ALL_CAPS`. + This is a more tolerant policy than the [C++ Core + Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Renum-caps), + which recommend using `snake_case`. Please use what seems appropriate. - Class names, function names, and method names are UpperCamelCase - (PascalCase). Do not prefix class names with `C`. + (PascalCase). Do not prefix class names with `C`. See [Internal interface + naming style](#internal-interface-naming-style) for an exception to this + convention. + - Test suite naming convention: The Boost test suite in file `src/test/foo_tests.cpp` should be named `foo_tests`. Test suite names must be unique. @@ -100,6 +110,28 @@ code. - `nullptr` is preferred over `NULL` or `(void*)0`. - `static_assert` is preferred over `assert` where possible. Generally; compile-time checking is preferred over run-time checking. +For function calls a namespace should be specified explicitly, unless such functions have been declared within it. +Otherwise, [argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl), also known as ADL, could be +triggered that makes code harder to maintain and reason about: +```c++ +#include + +namespace fs { +class path : public std::filesystem::path +{ +}; +// The intention is to disallow this function. +bool exists(const fs::path& p) = delete; +} // namespace fs + +int main() +{ + //fs::path p; // error + std::filesystem::path p; // compiled + exists(p); // ADL being used for unqualified name lookup +} +``` + Block style example: ```c++ int g_count = 0; @@ -132,11 +164,78 @@ public: } // namespace foo ``` +Coding Style (C++ functions and methods) +-------------------- + +- When ordering function parameters, place input parameters first, then any + in-out parameters, followed by any output parameters. + +- *Rationale*: API consistency. + +- Prefer returning values directly to using in-out or output parameters. Use + `std::optional` where helpful for returning values. + +- *Rationale*: Less error-prone (no need for assumptions about what the output + is initialized to on failure), easier to read, and often the same or better + performance. + +- Generally, use `std::optional` to represent optional by-value inputs (and + instead of a magic default value, if there is no real default). Non-optional + input parameters should usually be values or const references, while + non-optional in-out and output parameters should usually be references, as + they cannot be null. + +Coding Style (C++ named arguments) +------------------------------ + +When passing named arguments, use a format that clang-tidy understands. The +argument names can otherwise not be verified by clang-tidy. + +For example: + +```c++ +void function(Addrman& addrman, bool clear); + +int main() +{ + function(g_addrman, /*clear=*/false); +} +``` + +### Running clang-tidy + +To run clang-tidy on Ubuntu/Debian, install the dependencies: + +```sh +apt install clang-tidy bear clang +``` + +Then, pass clang as compiler to configure, and use bear to produce the `compile_commands.json`: + +```sh +./autogen.sh && ./configure CC=clang CXX=clang++ +make clean && bear make -j $(nproc) # For bear 2.x +make clean && bear -- make -j $(nproc) # For bear 3.x +``` + +To run clang-tidy on all source files: + +```sh +( cd ./src/ && run-clang-tidy -j $(nproc) ) +``` + +To run clang-tidy on the changed source lines: + +```sh +git diff | ( cd ./src/ && clang-tidy-diff -p2 -j $(nproc) ) +``` + Coding Style (Python) --------------------- Refer to [/test/functional/README.md#style-guidelines](/test/functional/README.md#style-guidelines). + Coding Style (Doxygen-compatible comments) ------------------------------------------ @@ -249,6 +348,35 @@ Development tips and tricks Run configure with `--enable-debug` to add additional compiler flags that produce better debugging builds. +### Show sources in debugging + +If you have ccache enabled, absolute paths are stripped from debug information +with the -fdebug-prefix-map and -fmacro-prefix-map options (if supported by the +compiler). This might break source file detection in case you move binaries +after compilation, debug from the directory other than the project root or use +an IDE that only supports absolute paths for debugging. + +There are a few possible fixes: + +1. Configure source file mapping. + +For `gdb` create or append to `.gdbinit` file: +``` +set substitute-path ./src /path/to/project/root/src +``` + +For `lldb` create or append to `.lldbinit` file: +``` +settings set target.source-map ./src /path/to/project/root/src +``` + +2. Add a symlink to the `./src` directory: +``` +ln -s /path/to/project/root/src src +``` + +3. Use `debugedit` to modify debug information in the binary. + ### Compiling for gprof profiling Run configure with the `--enable-gprof` option, then make. @@ -258,8 +386,10 @@ Run configure with the `--enable-gprof` option, then make. If the code is behaving strangely, take a look in the `debug.log` file in the data directory; error and debugging messages are written there. -The `-debug=...` command-line option controls debugging; running with just `-debug` or `-debug=1` will turn -on all categories (and give you a very large `debug.log` file). +Debug logging can be enabled on startup with the `-debug` and `-loglevel` +configuration options and toggled while bitcoind is running with the `logging` +RPC. For instance, launching bitcoind with `-debug` or `-debug=1` will turn on +all log categories and `-loglevel=trace` will turn on all log severity levels. The Qt code routes `qDebug()` output to `debug.log` under category "qt": run with `-debug=qt` to see it. @@ -282,6 +412,21 @@ configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts run-time checks to keep track of which locks are held and adds warnings to the `debug.log` file if inconsistencies are detected. +### DEBUG_LOCKCONTENTION + +Defining `DEBUG_LOCKCONTENTION` adds a "lock" logging category to the logging +RPC that, when enabled, logs the location and duration of each lock contention +to the `debug.log` file. + +The `--enable-debug` configure option adds `-DDEBUG_LOCKCONTENTION` to the +compiler flags. You may also enable it manually for a non-debug build by running +configure with `-DDEBUG_LOCKCONTENTION` added to your CPPFLAGS, +i.e. `CPPFLAGS="-DDEBUG_LOCKCONTENTION"`, then build and run bitcoind. + +You can then use the `-debug=lock` configuration option at bitcoind startup or +`bitcoin-cli logging '["lock"]'` at runtime to turn on lock contention logging. +It can be toggled off again with `bitcoin-cli logging [] '["lock"]'`. + ### Assertions and Checks The util file `src/util/check.h` offers helpers to protect against coding and @@ -297,7 +442,7 @@ other input. failure, it will throw an exception, which can be caught to recover from the error. - For example, a nullptr dereference or any other logic bug in RPC code - means that the RPC code is faulty and can not be executed. However, the + means that the RPC code is faulty and cannot be executed. However, the logic bug can be shown to the user and the program can continue to run. * `Assume` should be used to document assumptions when program execution can safely continue even if the assumption is violated. In debug builds it @@ -345,7 +490,7 @@ make cov Profiling is a good way to get a precise idea of where time is being spent in code. One tool for doing profiling on Linux platforms is called -[`perf`](http://www.brendangregg.com/perf.html), and has been integrated into +[`perf`](https://www.brendangregg.com/perf.html), and has been integrated into the functional test framework. Perf can observe a running process and sample (at some frequency) where its execution is. @@ -460,10 +605,10 @@ Threads : Started from `main()` in `bitcoind.cpp`. Responsible for starting up and shutting down the application. -- [ThreadImport (`b-loadblk`)](https://doxygen.bitcoincore.org/init_8cpp.html#ae9e290a0e829ec0198518de2eda579d1) +- [ThreadImport (`b-loadblk`)](https://doxygen.bitcoincore.org/namespacenode.html#ab4305679079866f0f420f7dbf278381d) : Loads blocks from `blk*.dat` files or `-loadblock=` on startup. -- [ThreadScriptCheck (`b-scriptch.x`)](https://doxygen.bitcoincore.org/validation_8cpp.html#a925a33e7952a157922b0bbb8dab29a20) +- [CCheckQueue::Loop (`b-scriptch.x`)](https://doxygen.bitcoincore.org/class_c_check_queue.html#a6e7fa51d3a25e7cb65446d4b50e6a987) : Parallel script validation threads for transactions in blocks. - [ThreadHTTP (`b-http`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#abb9f6ea8819672bd9a62d3695070709c) @@ -479,7 +624,7 @@ Threads : Does asynchronous background tasks like dumping wallet contents, dumping addrman and running asynchronous validationinterface callbacks. -- [TorControlThread (`b-torcontrol`)](https://doxygen.bitcoincore.org/torcontrol_8cpp.html#a4faed3692d57a0d7bdbecf3b37f72de0) +- [TorControlThread (`b-torcontrol`)](https://doxygen.bitcoincore.org/torcontrol_8cpp.html#a52a3efff23634500bb42c6474f306091) : Libevent thread for tor connections. - Net threads: @@ -491,7 +636,7 @@ Threads - [ThreadDNSAddressSeed (`b-dnsseed`)](https://doxygen.bitcoincore.org/class_c_connman.html#aa7c6970ed98a4a7bafbc071d24897d13) : Loads addresses of peers from the DNS. - - [ThreadMapPort (`b-upnp`)](https://doxygen.bitcoincore.org/net_8cpp.html#a63f82a71c4169290c2db1651a9bbe249) + - ThreadMapPort (`b-mapport`) : Universal plug-and-play startup/shutdown. - [ThreadSocketHandler (`b-net`)](https://doxygen.bitcoincore.org/class_c_connman.html#a765597cbfe99c083d8fa3d61bb464e34) @@ -503,6 +648,9 @@ Threads - [ThreadOpenConnections (`b-opencon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a55e9feafc3bab78e5c9d408c207faa45) : Initiates new connections to peers. + - [ThreadI2PAcceptIncoming (`b-i2paccept`)](https://doxygen.bitcoincore.org/class_c_connman.html#a57787b4f9ac847d24065fbb0dd6e70f8) + : Listens for and accepts incoming I2P connections through the I2P SAM proxy. + Ignoring IDE/editor files -------------------------- @@ -563,10 +711,6 @@ Wallet - Make sure that no crashes happen with run-time option `-disablewallet`. -- Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set. - - - *Rationale*: Otherwise compilation of the disable-wallet build will fail in environments without BerkeleyDB. - General C++ ------------- @@ -669,19 +813,19 @@ Foo(vec); ```cpp enum class Tabs { - INFO, - CONSOLE, - GRAPH, - PEERS + info, + console, + network_graph, + peers }; int GetInt(Tabs tab) { switch (tab) { - case Tabs::INFO: return 0; - case Tabs::CONSOLE: return 1; - case Tabs::GRAPH: return 2; - case Tabs::PEERS: return 3; + case Tabs::info: return 0; + case Tabs::console: return 1; + case Tabs::network_graph: return 2; + case Tabs::peers: return 3; } // no default case, so the compiler can warn about missing cases assert(false); } @@ -692,11 +836,6 @@ int GetInt(Tabs tab) Strings and formatting ------------------------ -- Be careful of `LogPrint` versus `LogPrintf`. `LogPrint` takes a `category` argument, `LogPrintf` does not. - - - *Rationale*: Confusion of these can result in runtime exceptions due to - formatting mismatch, and it is easy to get wrong because of subtly similar naming. - - Use `std::string`, avoid C string manipulation functions. - *Rationale*: C++ string handling is marginally safer, less scope for @@ -769,22 +908,37 @@ from using a different variable with the same name), please name variables so that their names do not shadow variables defined in the source code. When using nested cycles, do not name the inner cycle variable the same as in -the upper cycle, etc. +the outer cycle, etc. + +Lifetimebound +-------------- + +The [Clang `lifetimebound` +attribute](https://clang.llvm.org/docs/AttributeReference.html#lifetimebound) +can be used to tell the compiler that a lifetime is bound to an object and +potentially see a compile-time warning if the object has a shorter lifetime from +the invalid use of a temporary. You can use the attribute by adding a `LIFETIMEBOUND` +annotation defined in `src/attributes.h`; please grep the codebase for examples. Threads and synchronization ---------------------------- -- Prefer `Mutex` type to `RecursiveMutex` one +- Prefer `Mutex` type to `RecursiveMutex` one. - Consistently use [Clang Thread Safety Analysis](https://clang.llvm.org/docs/ThreadSafetyAnalysis.html) annotations to - get compile-time warnings about potential race conditions in code. Combine annotations in function declarations with - run-time asserts in function definitions: + get compile-time warnings about potential race conditions or deadlocks in code. - In functions that are declared separately from where they are defined, the thread safety annotations should be added exclusively to the function declaration. Annotations on the definition could lead to false positives (lack of compile failure) at call sites between the two. + - Prefer locks that are in a class rather than global, and that are + internal to a class (private or protected) rather than public. + + - Combine annotations in function declarations with run-time asserts in + function definitions: + ```C++ // txmempool.h class CTxMemPool @@ -808,21 +962,37 @@ void CTxMemPool::UpdateTransactionsFromBlock(...) ```C++ // validation.h -class ChainstateManager +class Chainstate { +protected: + ... + Mutex m_chainstate_mutex; + ... public: ... - bool ProcessNewBlock(...) LOCKS_EXCLUDED(::cs_main); + bool ActivateBestChain( + BlockValidationState& state, + std::shared_ptr pblock = nullptr) + EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex) + LOCKS_EXCLUDED(::cs_main); + ... + bool PreciousBlock(BlockValidationState& state, CBlockIndex* pindex) + EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex) + LOCKS_EXCLUDED(::cs_main); ... } // validation.cpp -bool ChainstateManager::ProcessNewBlock(...) +bool Chainstate::PreciousBlock(BlockValidationState& state, CBlockIndex* pindex) { + AssertLockNotHeld(m_chainstate_mutex); AssertLockNotHeld(::cs_main); - ... - LOCK(::cs_main); - ... + { + LOCK(cs_main); + ... + } + + return ActivateBestChain(state, std::shared_ptr()); } ``` @@ -853,6 +1023,8 @@ TRY_LOCK(cs_vNodes, lockNodes); Scripts -------------------------- +Write scripts in Python rather than bash, when possible. + ### Shebang - Use `#!/usr/bin/env bash` instead of obsolete `#!/bin/bash`. @@ -959,37 +1131,40 @@ Subtrees Several parts of the repository are subtrees of software maintained elsewhere. -Some of these are maintained by active developers of Bitcoin Core, in which case changes should probably go -directly upstream without being PRed directly against the project. They will be merged back in the next -subtree merge. +Some of these are maintained by active developers of Bitcoin Core, in which case +changes should go directly upstream without being PRed directly against the project. +They will be merged back in the next subtree merge. -Others are external projects without a tight relationship with our project. Changes to these should also -be sent upstream, but bugfixes may also be prudent to PR against Bitcoin Core so that they can be integrated -quickly. Cosmetic changes should be purely taken upstream. +Others are external projects without a tight relationship with our project. Changes +to these should also be sent upstream, but bugfixes may also be prudent to PR against +a Bitcoin Core subtree, so that they can be integrated quickly. Cosmetic changes +should be taken upstream. -There is a tool in `test/lint/git-subtree-check.sh` ([instructions](../test/lint#git-subtree-checksh)) to check a subtree directory for consistency with -its upstream repository. +There is a tool in `test/lint/git-subtree-check.sh` ([instructions](../test/lint#git-subtree-checksh)) +to check a subtree directory for consistency with its upstream repository. Current subtrees include: - src/leveldb - - Upstream at https://github.com/google/leveldb ; Maintained by Google, but - open important PRs to Core to avoid delay. + - Subtree at https://github.com/bitcoin-core/leveldb-subtree ; maintained by Core contributors. + - Upstream at https://github.com/google/leveldb ; maintained by Google. Open + important PRs to the subtree to avoid delay. - **Note**: Follow the instructions in [Upgrading LevelDB](#upgrading-leveldb) when merging upstream changes to the LevelDB subtree. - src/crc32c - Used by leveldb for hardware acceleration of CRC32C checksums for data integrity. - - Upstream at https://github.com/google/crc32c ; Maintained by Google. + - Subtree at https://github.com/bitcoin-core/crc32c-subtree ; maintained by Core contributors. + - Upstream at https://github.com/google/crc32c ; maintained by Google. - src/secp256k1 - - Upstream at https://github.com/bitcoin-core/secp256k1/ ; actively maintained by Core contributors. + - Upstream at https://github.com/bitcoin-core/secp256k1/ ; maintained by Core contributors. - src/crypto/ctaes - - Upstream at https://github.com/bitcoin-core/ctaes ; actively maintained by Core contributors. + - Upstream at https://github.com/bitcoin-core/ctaes ; maintained by Core contributors. -- src/univalue - - Upstream at https://github.com/bitcoin-core/univalue ; actively maintained by Core contributors, deviates from upstream https://github.com/jgarzik/univalue +- src/minisketch + - Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors. Upgrading LevelDB --------------------- @@ -1119,7 +1294,10 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: Consistency with the existing interface. -- Argument naming: use snake case `fee_delta` (and not, e.g. camel case `feeDelta`) +- Argument and field naming: please consider whether there is already a naming + style or spelling convention in the API for the type of object in question + (`blockhash`, for example), and if so, try to use that. If not, use snake case + `fee_delta` (and not, e.g. `feedelta` or camel case `feeDelta`). - *Rationale*: Consistency with the existing interface. @@ -1158,7 +1336,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: - Don't forget to fill in the argument names correctly in the RPC command table. - - *Rationale*: If not, the call can not be used with name-based arguments. + - *Rationale*: If not, the call cannot be used with name-based arguments. - Add every non-string RPC argument `(method, idx, name)` to the table `vRPCConvertParams` in `rpc/client.cpp`. @@ -1213,6 +1391,12 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: User-facing consistency. +- Use `fs::path::u8string()` and `fs::u8path()` functions when converting path + to JSON strings, not `fs::PathToString` and `fs::PathFromString` + + - *Rationale*: JSON strings are Unicode strings, not byte strings, and + RFC8259 requires JSON to be encoded as UTF-8. + Internal interface guidelines ----------------------------- @@ -1279,22 +1463,9 @@ communication: virtual boost::signals2::scoped_connection connectTipChanged(TipChangedFn fn) = 0; ``` -- For consistency and friendliness to code generation tools, interface method - input and inout parameters should be ordered first and output parameters - should come last. +- Interface methods should not be overloaded. - Example: - - ```c++ - // Good: error output param is last - virtual bool broadcastTransaction(const CTransactionRef& tx, CAmount max_fee, std::string& error) = 0; - - // Bad: error output param is between input params - virtual bool broadcastTransaction(const CTransactionRef& tx, std::string& error, CAmount max_fee) = 0; - ``` - -- For friendliness to code generation tools, interface methods should not be - overloaded: + *Rationale*: consistency and friendliness to code generation tools. Example: @@ -1308,10 +1479,13 @@ communication: virtual bool disconnect(NodeId id) = 0; ``` -- For consistency and friendliness to code generation tools, interface method - names should be `lowerCamelCase` and standalone function names should be +### Internal interface naming style + +- Interface method names should be `lowerCamelCase` and standalone function names should be `UpperCamelCase`. + *Rationale*: consistency and friendliness to code generation tools. + Examples: ```c++ diff --git a/doc/files.md b/doc/files.md index e670d77ae5d2..f88d3f91a1c1 100644 --- a/doc/files.md +++ b/doc/files.md @@ -56,7 +56,6 @@ Subdirectory | File(s) | Description `indexes/coinstats/db/` | LevelDB database | Coinstats index; *optional*, used if `-coinstatsindex=1` `wallets/` | | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, wallets reside in the [data directory](#data-directory-location) `./` | `anchors.dat` | Anchor IP address database, created on shutdown and deleted at startup. Anchors are last known outgoing block-relay-only peers that are tried to re-connect to on startup -`./` | `banlist.dat` | Stores the addresses/subnets of banned nodes (deprecated). `bitcoind` or `bitcoin-qt` no longer save the banlist to this file, but read it on startup if `banlist.json` is not present. `./` | `banlist.json` | Stores the addresses/subnets of banned nodes. `./` | `bitcoin.conf` | User-defined [configuration settings](bitcoin-conf.md) for `bitcoind` or `bitcoin-qt`. File is not written to by the software and must be created manually. Path can be specified by `-conf` option `./` | `bitcoind.pid` | Stores the process ID (PID) of `bitcoind` or `bitcoin-qt` while running; created at start and deleted on shutdown; can be specified by `-pid` option @@ -114,6 +113,7 @@ These subdirectories and files are no longer used by Bitcoin Core: Path | Description | Repository notes ---------------|-------------|----------------- +`banlist.dat` | Stores the addresses/subnets of banned nodes; superseded by `banlist.json` in 22.0 and completely ignored in 23.0 | [PR #20966](https://github.com/bitcoin/bitcoin/pull/20966), [PR #22570](https://github.com/bitcoin/bitcoin/pull/22570) `blktree/` | Blockchain index; replaced by `blocks/index/` in [0.8.0](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.8.0.md#improvements) | [PR #2231](https://github.com/bitcoin/bitcoin/pull/2231), [`8fdc94cc`](https://github.com/bitcoin/bitcoin/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15) `coins/` | Unspent transaction output database; replaced by `chainstate/` in 0.8.0 | [PR #2231](https://github.com/bitcoin/bitcoin/pull/2231), [`8fdc94cc`](https://github.com/bitcoin/bitcoin/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15) `blkindex.dat` | Blockchain index BDB database; replaced by {`chainstate/`, `blocks/index/`, `blocks/revNNNNN.dat`[\[2\]](#note2)} in 0.8.0 | [PR #1677](https://github.com/bitcoin/bitcoin/pull/1677) diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 6fc9077e4c3b..9abfbc9213ec 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -16,6 +16,13 @@ $ FUZZ=process_message src/test/fuzz/fuzz # abort fuzzing using ctrl-c ``` +There is also a runner script to execute all fuzz targets. Refer to +`./test/fuzz/test_runner.py --help` for more details. + +## Overview of Bitcoin Core fuzzing + +[Google](https://github.com/google/fuzzing/) has a good overview of fuzzing in general, with contributions from key architects of some of the most-used fuzzers. [This paper](https://agroce.github.io/bitcoin_report.pdf) includes an external overview of the status of Bitcoin Core fuzzing, as of summer 2021. [John Regehr](https://blog.regehr.org/archives/1687) provides good advice on writing code that assists fuzzers in finding bugs, which is useful for developers to keep in mind. + ## Fuzzing harnesses and output [`process_message`](https://github.com/bitcoin/bitcoin/blob/master/src/test/fuzz/process_message.cpp) is a fuzzing harness for the [`ProcessMessage(...)` function (`net_processing`)](https://github.com/bitcoin/bitcoin/blob/master/src/net_processing.cpp). The available fuzzing harnesses are found in [`src/test/fuzz/`](https://github.com/bitcoin/bitcoin/tree/master/src/test/fuzz). @@ -64,6 +71,15 @@ block^@M-^?M-^?M-^?M-^?M-^?nM-^?M-^? In this case the fuzzer managed to create a `block` message which when passed to `ProcessMessage(...)` increased coverage. +It is possible to specify `bitcoind` arguments to the `fuzz` executable. +Depending on the test, they may be ignored or consumed and alter the behavior +of the test. Just make sure to use double-dash to distinguish them from the +fuzzer's own arguments: + +```sh +$ FUZZ=address_deserialize_v2 src/test/fuzz/fuzz -runs=1 fuzz_seed_corpus/address_deserialize_v2 --checkaddrman=5 --printtoconsole=1 +``` + ## Fuzzing corpora The project's collection of seed corpora is found in the [`bitcoin-core/qa-assets`](https://github.com/bitcoin-core/qa-assets) repo. @@ -83,6 +99,10 @@ INFO: seed corpus: files: 991 min: 1b max: 1858b total: 288291b rss: 150Mb … ``` +## Run without sanitizers for increased throughput + +Fuzzing on a harness compiled with `--with-sanitizers=address,fuzzer,undefined` is good for finding bugs. However, the very slow execution even under libFuzzer will limit the ability to find new coverage. A good approach is to perform occasional long runs without the additional bug-detectors (configure `--with-sanitizers=fuzzer`) and then merge new inputs into a corpus as described in the qa-assets repo (https://github.com/bitcoin-core/qa-assets/blob/main/.github/PULL_REQUEST_TEMPLATE.md). Patience is useful; even with improved throughput, libFuzzer may need days and 10s of millions of executions to reach deep/hard targets. + ## Reproduce a fuzzer crash reported by the CI - `cd` into the `qa-assets` directory and update it with `git pull qa-assets` @@ -247,6 +267,73 @@ $ honggfuzz/honggfuzz --exit_upon_crash --quiet --timeout 4 -n 1 -Q \ -debug ``` +# Fuzzing Bitcoin Core using Eclipser (v1.x) + +## Quickstart guide + +To quickly get started fuzzing Bitcoin Core using [Eclipser v1.x](https://github.com/SoftSec-KAIST/Eclipser/tree/v1.x): + +```sh +$ git clone https://github.com/bitcoin/bitcoin +$ cd bitcoin/ +$ sudo vim /etc/apt/sources.list # Uncomment the lines starting with 'deb-src'. +$ sudo apt-get update +$ sudo apt-get build-dep qemu +$ sudo apt-get install libtool libtool-bin wget automake autoconf bison gdb +``` + +At this point, you must install the .NET core. The process differs, depending on your Linux distribution. +See [this link](https://docs.microsoft.com/en-us/dotnet/core/install/linux) for details. +On ubuntu 20.04, the following should work: + +```sh +$ wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb +$ sudo dpkg -i packages-microsoft-prod.deb +$ rm packages-microsoft-prod.deb +$ sudo apt-get update +$ sudo apt-get install -y dotnet-sdk-2.1 +``` + +You will also want to make sure Python is installed as `python` for the Eclipser install to succeed. + +```sh +$ git clone https://github.com/SoftSec-KAIST/Eclipser.git +$ cd Eclipser +$ git checkout v1.x +$ make +$ cd .. +$ ./autogen.sh +$ ./configure --enable-fuzz +$ make +$ mkdir -p outputs/ +$ FUZZ=bech32 dotnet Eclipser/build/Eclipser.dll fuzz -p src/test/fuzz/fuzz -t 36000 -o outputs --src stdin +``` + +This will perform 10 hours of fuzzing. + +To make further use of the inputs generated by Eclipser, you +must first decode them: + +```sh +$ dotnet Eclipser/build/Eclipser.dll decode -i outputs/testcase -o decoded_outputs +``` +This will place raw inputs in the directory `decoded_outputs/decoded_stdins`. Crashes are in the `outputs/crashes` directory, and must +be decoded in the same way. + +Fuzzing with Eclipser will likely be much more effective if using an existing corpus: + +```sh +$ git clone https://github.com/bitcoin-core/qa-assets +$ FUZZ=bech32 dotnet Eclipser/build/Eclipser.dll fuzz -p src/test/fuzz/fuzz -t 36000 -i qa-assets/fuzz_seed_corpus/bech32 outputs --src stdin +``` + +Note that fuzzing with Eclipser on certain targets (those that create 'full nodes', e.g. `process_message*`) will, +for now, slowly fill `/tmp/` with improperly cleaned-up files, which will cause spurious crashes. +See [this proposed patch](https://github.com/bitcoin/bitcoin/pull/22472) for more information. + +Read the [Eclipser documentation for v1.x](https://github.com/SoftSec-KAIST/Eclipser/tree/v1.x) for more details on using Eclipser. + + # OSS-Fuzz Bitcoin Core participates in Google's [OSS-Fuzz](https://github.com/google/oss-fuzz/tree/master/projects/bitcoin-core) diff --git a/doc/gitian-building.md b/doc/gitian-building.md deleted file mode 100644 index 3a48f4a0b397..000000000000 --- a/doc/gitian-building.md +++ /dev/null @@ -1,4 +0,0 @@ -Gitian building -================ - -This file was moved to [the Bitcoin Core documentation repository](https://github.com/bitcoin-core/docs/blob/master/gitian-building.md) at [https://github.com/bitcoin-core/docs](https://github.com/bitcoin-core/docs). diff --git a/doc/i2p.md b/doc/i2p.md index 27ef4d9d9f0d..8433bbeaa292 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -10,11 +10,22 @@ started with I2P terminology. ## Run Bitcoin Core with an I2P router (proxy) A running I2P router (proxy) with [SAM](https://geti2p.net/en/docs/api/samv3) -enabled is required (there is an [official one](https://geti2p.net) and -[a few alternatives](https://en.wikipedia.org/wiki/I2P#Routers)). Notice the IP -address and port the SAM proxy is listening to; usually, it is -`127.0.0.1:7656`. Once it is up and running with SAM enabled, use the following -Bitcoin Core options: +enabled is required. Options include: + +- [i2prouter (I2P Router)](https://geti2p.net), the official implementation in + Java +- [i2pd (I2P Daemon)](https://github.com/PurpleI2P/i2pd) + ([documentation](https://i2pd.readthedocs.io/en/latest)), a lighter + alternative in C++ (successfully tested with version 2.23 and up; version 2.36 + or later recommended) +- [i2p-zero](https://github.com/i2p-zero/i2p-zero) +- [other alternatives](https://en.wikipedia.org/wiki/I2P#Routers) + +Note the IP address and port the SAM proxy is listening to; usually, it is +`127.0.0.1:7656`. + +Once an I2P router with SAM enabled is up and running, use the following Bitcoin +Core configuration options: ``` -i2psam= @@ -36,29 +47,66 @@ In a typical situation, this suffices: bitcoind -i2psam=127.0.0.1:7656 ``` -The first time Bitcoin Core connects to the I2P router, its I2P address (and -corresponding private key) will be automatically generated and saved in a file -named `i2p_private_key` in the Bitcoin Core data directory. +The first time Bitcoin Core connects to the I2P router, if +`-i2pacceptincoming=1`, then it will automatically generate a persistent I2P +address and its corresponding private key. The private key will be saved in a +file named `i2p_private_key` in the Bitcoin Core data directory. The persistent +I2P address is used for accepting incoming connections and for making outgoing +connections if `-i2pacceptincoming=1`. If `-i2pacceptincoming=0` then only +outbound I2P connections are made and a different transient I2P address is used +for each connection to improve privacy. + +## Persistent vs transient I2P addresses + +In I2P connections, the connection receiver sees the I2P address of the +connection initiator. This is unlike the Tor network where the recipient does +not know who is connecting to them and can't tell if two connections are from +the same peer or not. + +If an I2P node is not accepting incoming connections, then Bitcoin Core uses +random, one-time, transient I2P addresses for itself for outbound connections +to make it harder to discriminate, fingerprint or analyze it based on its I2P +address. ## Additional configuration options related to I2P -You may set the `debug=i2p` config logging option to have additional -information in the debug log about your I2P configuration and connections. Run -`bitcoin-cli help logging` for more information. +``` +-debug=i2p +``` + +Set the `debug=i2p` config logging option to see additional information in the +debug log about your I2P configuration and connections. Run `bitcoin-cli help +logging` for more information. + +``` +-onlynet=i2p +``` + +Make automatic outbound connections only to I2P addresses. Inbound and manual +connections are not affected by this option. It can be specified multiple times +to allow multiple networks, e.g. onlynet=onion, onlynet=i2p. -It is possible to restrict outgoing connections in the usual way with -`onlynet=i2p`. I2P support was added to Bitcoin Core in version 22.0 (mid 2021) -and there may be fewer I2P peers than Tor or IP ones. Therefore, using -`onlynet=i2p` alone (without other `onlynet=`) may make a node more susceptible -to [Sybil attacks](https://en.bitcoin.it/wiki/Weaknesses#Sybil_attack). Use +I2P support was added to Bitcoin Core in version 22.0 and there may be fewer I2P +peers than Tor or IP ones. Therefore, using I2P alone without other networks may +make a node more susceptible to [Sybil +attacks](https://en.bitcoin.it/wiki/Weaknesses#Sybil_attack). You can use `bitcoin-cli -addrinfo` to see the number of I2P addresses known to your node. -## I2P related information in Bitcoin Core +Another consideration with `onlynet=i2p` is that the initial blocks download +phase when syncing up a new node can be very slow. This phase can be sped up by +using other networks, for instance `onlynet=onion`, at the same time. + +In general, a node can be run with both onion and I2P hidden services (or +any/all of IPv4/IPv6/onion/I2P/CJDNS), which can provide a potential fallback if +one of the networks has issues. + +## I2P-related information in Bitcoin Core -There are several ways to see your I2P address in Bitcoin Core: -- in the debug log (grep for `AddLocal`, the I2P address ends in `.b32.i2p`) -- in the output of the `getnetworkinfo` RPC in the "localaddresses" section -- in the output of `bitcoin-cli -netinfo` peer connections dashboard +There are several ways to see your I2P address in Bitcoin Core if accepting +incoming I2P connections (`-i2pacceptincoming`): +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` +- in the debug log (grep for `AddLocal`; the I2P address ends in `.b32.i2p`) To see which I2P peers your node is connected to, use `bitcoin-cli -netinfo 4` or the `getpeerinfo` RPC (e.g. `bitcoin-cli getpeerinfo`). @@ -85,3 +133,29 @@ listening port to 0 when listening for incoming I2P connections and advertises its own I2P address with port 0. Furthermore, it will not attempt to connect to I2P addresses with a non-zero port number because with SAM v3.1 the destination port (`TO_PORT`) is always set to 0 and is not in the control of Bitcoin Core. + +## Bandwidth + +I2P routers may route a large amount of general network traffic with their +default settings. Check your router's configuration to limit the amount of this +traffic relayed, if desired. + +With `i2pd`, the amount of bandwidth being shared with the wider network can be +adjusted with the `bandwidth`, `share` and `transittunnels` options in your +`i2pd.conf` file. For example, to limit total I2P traffic to 256KB/s and share +50% of this limit for a maximum of 20 transit tunnels: + +``` +bandwidth = 256 +share = 50 + +[limits] +transittunnels = 20 +``` + +If you prefer not to relay any public I2P traffic and only permit I2P traffic +from programs which are connecting via the SAM proxy, e.g. Bitcoin Core, you +can set the `notransit` option to `true`. + +Similar bandwidth configuration options for the Java I2P router can be found in +`http://127.0.0.1:7657/config` under the "Bandwidth" tab. diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md new file mode 100644 index 000000000000..366d7ec54bee --- /dev/null +++ b/doc/managing-wallets.md @@ -0,0 +1,148 @@ +# Managing the Wallet + +## 1. Backing Up and Restoring The Wallet + +### 1.1 Creating the Wallet + +Since version 0.21, Bitcoin Core no longer has a default wallet. +Wallets can be created with the `createwallet` RPC or with the `Create wallet` GUI menu item. + +In the GUI, the `Create a new wallet` button is displayed on the main screen when there is no wallet loaded. Alternatively, there is the option `File` ->`Create wallet`. + +The following command, for example, creates a descriptor wallet. More information about this command may be found by running `bitcoin-cli help createwallet`. + +``` +$ bitcoin-cli createwallet "wallet-01" +``` + +By default, wallets are created in the `wallets` folder of the data directory, which varies by operating system, as shown below. The user can change the default by using the `-datadir` or `-walletdir` initialization parameters. + +| Operating System | Default wallet directory | +| -----------------|:------------------------------------------------------------| +| Linux | `/home//.bitcoin/wallets` | +| Windows | `C:\Users\\AppData\Roaming\Bitcoin\wallets` | +| macOS | `/Users//Library/Application Support/Bitcoin/wallets` | + +### 1.2 Encrypting the Wallet + +The `wallet.dat` file is not encrypted by default and is, therefore, vulnerable if an attacker gains access to the device where the wallet or the backups are stored. + +Wallet encryption may prevent unauthorized access. However, this significantly increases the risk of losing coins due to forgotten passphrases. There is no way to recover a passphrase. This tradeoff should be well thought out by the user. + +Wallet encryption may also not protect against more sophisticated attacks. An attacker can, for example, obtain the password by installing a keylogger on the user's machine. + +After encrypting the wallet or changing the passphrase, a new backup needs to be created immediately. The reason is that the keypool is flushed and a new HD seed is generated after encryption. Any bitcoins received by the new seed cannot be recovered from the previous backups. + +The wallet's private key may be encrypted with the following command: + +``` +$ bitcoin-cli -rpcwallet="wallet-01" encryptwallet "passphrase" +``` + +Once encrypted, the passphrase can be changed with the `walletpassphrasechange` command. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" walletpassphrasechange "oldpassphrase" "newpassphrase" +``` + +The argument passed to `-rpcwallet` is the name of the wallet to be encrypted. + +Only the wallet's private key is encrypted. All other wallet information, such as transactions, is still visible. + +The wallet's private key can also be encrypted in the `createwallet` command via the `passphrase` argument: + +``` +$ bitcoin-cli -named createwallet wallet_name="wallet-01" passphrase="passphrase" +``` + +Note that if the passphrase is lost, all the coins in the wallet will also be lost forever. + +### 1.3 Unlocking the Wallet + +If the wallet is encrypted and the user tries any operation related to private keys, such as sending bitcoins, an error message will be displayed. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" sendtoaddress "tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx" 0.01 +error code: -13 +error message: +Error: Please enter the wallet passphrase with walletpassphrase first. +``` + +To unlock the wallet and allow it to run these operations, the `walletpassphrase` RPC is required. + +This command takes the passphrase and an argument called `timeout`, which specifies the time in seconds that the wallet decryption key is stored in memory. After this period expires, the user needs to execute this RPC again. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" walletpassphrase "passphrase" 120 +``` + +In the GUI, there is no specific menu item to unlock the wallet. When the user sends bitcoins, the passphrase will be prompted automatically. + +### 1.4 Backing Up the Wallet + +To backup the wallet, the `backupwallet` RPC or the `Backup Wallet` GUI menu item must be used to ensure the file is in a safe state when the copy is made. + +In the RPC, the destination parameter must include the name of the file. Otherwise, the command will return an error message like "Error: Wallet backup failed!" for descriptor wallets. If it is a legacy wallet, it will be copied and a file will be created with the default file name `wallet.dat`. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" backupwallet /home/node01/Backups/backup-01.dat +``` + +In the GUI, the wallet is selected in the `Wallet` drop-down list in the upper right corner. If this list is not present, the wallet can be loaded in `File` ->`Open wallet` if necessary. Then, the backup can be done in `File` -> `Backup Wallet...`. + +This backup file can be stored on one or multiple offline devices, which must be reliable enough to work in an emergency and be malware free. Backup files can be regularly tested to avoid problems in the future. + +If the computer has malware, it can compromise the wallet when recovering the backup file. One way to minimize this is to not connect the backup to an online device. + +If both the wallet and all backups are lost for any reason, the bitcoins related to this wallet will become permanently inaccessible. + +### 1.5 Backup Frequency + +The original Bitcoin Core wallet was a collection of unrelated private keys. If a non-HD wallet had received funds to an address and then was restored from a backup made before the address was generated, then any funds sent to that address would have been lost because there was no deterministic mechanism to derive the address again. + +Bitcoin Core [version 0.13](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.13.0.md) introduced HD wallets with deterministic key derivation. With HD wallets, users no longer lose funds when restoring old backups because all addresses are derived from the HD wallet seed. + +This means that a single backup is enough to recover the coins at any time. It is still recommended to make regular backups (once a week) or after a significant number of new transactions to maintain the metadata, such as labels. Metadata cannot be retrieved from a blockchain rescan, so if the backup is too old, the metadata will be lost forever. + +Wallets created before version 0.13 are not HD and must be backed up every 100 keys used since the previous backup, or even more often to maintain the metadata. + +### 1.6 Restoring the Wallet From a Backup + +To restore a wallet, the `restorewallet` RPC must be used. + +``` +$ bitcoin-cli restorewallet "restored-wallet" /home/node01/Backups/backup-01.dat +``` + +After that, `getwalletinfo` can be used to check if the wallet has been fully restored. + +``` +$ bitcoin-cli -rpcwallet="restored-wallet" getwalletinfo +``` + +The restored wallet can also be loaded in the GUI via `File` ->`Open wallet`. + +## Migrating Legacy Wallets to Descriptor Wallets + +Legacy wallets (traditional non-descriptor wallets) can be migrated to become Descriptor wallets +through the use of the `migratewallet` RPC. Migrated wallets will have all of their addresses and private keys added to +a newly created Descriptor wallet that has the same name as the original wallet. Because Descriptor +wallets do not support having private keys and watch-only scripts, there may be up to two +additional wallets created after migration. In addition to a descriptor wallet of the same name, +there may also be a wallet named `_watchonly` and `_solvables`. `_watchonly` +contains all of the watchonly scripts. `_solvables` contains any scripts which the wallet +knows but is not watching the corresponding P2(W)SH scripts. + +Migrated wallets will also generate new addresses differently. While the same BIP 32 seed will be +used, the BIP 44, 49, 84, and 86 standard derivation paths will be used. After migrating, a new +backup of the wallet(s) will need to be created. + +Given that there is an extremely large number of possible configurations for the scripts that +Legacy wallets can know about, be watching for, and be able to sign for, `migratewallet` only +makes a best effort attempt to capture all of these things into Descriptor wallets. There may be +unforeseen configurations which result in some scripts being excluded. If a migration fails +unexpectedly or otherwise misses any scripts, please create an issue on GitHub. A backup of the +original wallet can be found in the wallet directory with the name `-.legacy.bak`. + +The backup can be restored using the `restorewallet` command as discussed in the +[Restoring the Wallet From a Backup](#16-restoring-the-wallet-from-a-backup) section diff --git a/doc/multisig-tutorial.md b/doc/multisig-tutorial.md new file mode 100644 index 000000000000..1d2b3244bca2 --- /dev/null +++ b/doc/multisig-tutorial.md @@ -0,0 +1,241 @@ +# 1. Multisig Tutorial + +Currently, it is possible to create a multisig wallet using Bitcoin Core only. + +Although there is already a brief explanation about the multisig in the [Descriptors documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md#multisig), this tutorial proposes to use the signet (instead of regtest), bringing the reader closer to a real environment and explaining some functions in more detail. + +This tutorial uses [jq](https://github.com/stedolan/jq) JSON processor to process the results from RPC and stores the relevant values in bash variables. This makes the tutorial reproducible and easier to follow step by step. + +Before starting this tutorial, start the bitcoin node on the signet network. + +```bash +./src/bitcoind -signet -daemon +``` + +This tutorial also uses the default WPKH derivation path to get the xpubs and does not conform to [BIP 45](https://github.com/bitcoin/bips/blob/master/bip-0045.mediawiki) or [BIP 87](https://github.com/bitcoin/bips/blob/master/bip-0087.mediawiki). + +At the time of writing, there is no way to extract a specific path from wallets in Bitcoin Core. For this, an external signer/xpub can be used. + +[PR #22341](https://github.com/bitcoin/bitcoin/pull/22341), which is still under development, introduces a new wallet RPC `getxpub`. It takes a BIP32 path as an argument and returns the xpub, along with the master key fingerprint. + +## 1.1 Basic Multisig Workflow + +### 1.1 Create the Descriptor Wallets + +For a 2-of-3 multisig, create 3 descriptor wallets. It is important that they are of the descriptor type in order to retrieve the wallet descriptors. These wallets contain HD seed and private keys, which will be used to sign the PSBTs and derive the xpub. + +These three wallets should not be used directly for privacy reasons (public key reuse). They should only be used to sign transactions for the (watch-only) multisig wallet. + +```bash +for ((n=1;n<=3;n++)) +do + ./src/bitcoin-cli -signet createwallet "participant_${n}" +done +``` + +Extract the xpub of each wallet. To do this, the `listdescriptors` RPC is used. By default, Bitcoin Core single-sig wallets are created using path `m/44'/1'/0'` for PKH, `m/84'/1'/0'` for WPKH, `m/49'/1'/0'` for P2WPKH-nested-in-P2SH and `m/86'/1'/0'` for P2TR based accounts. Each of them uses the chain 0 for external addresses and chain 1 for internal ones, as shown in the example below. + +``` +wpkh([1004658e/84'/1'/0']tpubDCBEcmVKbfC9KfdydyLbJ2gfNL88grZu1XcWSW9ytTM6fitvaRmVyr8Ddf7SjZ2ZfMx9RicjYAXhuh3fmLiVLPodPEqnQQURUfrBKiiVZc8/0/*)#g8l47ngv + +wpkh([1004658e/84'/1'/0']tpubDCBEcmVKbfC9KfdydyLbJ2gfNL88grZu1XcWSW9ytTM6fitvaRmVyr8Ddf7SjZ2ZfMx9RicjYAXhuh3fmLiVLPodPEqnQQURUfrBKiiVZc8/1/*)#en65rxc5 +``` + +The suffix (after #) is the checksum. Descriptors can optionally be suffixed with a checksum to protect against typos or copy-paste errors. +All RPCs in Bitcoin Core will include the checksum in their output. + +```bash +declare -A xpubs + +for ((n=1;n<=3;n++)) +do + xpubs["internal_xpub_${n}"]=$(./src/bitcoin-cli -signet -rpcwallet="participant_${n}" listdescriptors | jq '.descriptors | [.[] | select(.desc | startswith("wpkh") and contains("/1/*"))][0] | .desc' | grep -Po '(?<=\().*(?=\))') + + xpubs["external_xpub_${n}"]=$(./src/bitcoin-cli -signet -rpcwallet="participant_${n}" listdescriptors | jq '.descriptors | [.[] | select(.desc | startswith("wpkh") and contains("/0/*") )][0] | .desc' | grep -Po '(?<=\().*(?=\))') +done +``` + +`jq` is used to extract the xpub from the `wpkh` descriptor. + +The following command can be used to verify if the xpub was generated correctly. + +```bash +for x in "${!xpubs[@]}"; do printf "[%s]=%s\n" "$x" "${xpubs[$x]}" ; done +``` + +As previously mentioned, this step extracts the `m/84'/1'/0'` account instead of the path defined in [BIP 45](https://github.com/bitcoin/bips/blob/master/bip-0045.mediawiki) or [BIP 87](https://github.com/bitcoin/bips/blob/master/bip-0087.mediawiki), since there is no way to extract a specific path in Bitcoin Core at the time of writing. + +### 1.2 Define the Multisig Descriptors + +Define the external and internal multisig descriptors, add the checksum and then, join both in a JSON array. + +```bash +external_desc="wsh(sortedmulti(2,${xpubs["external_xpub_1"]},${xpubs["external_xpub_2"]},${xpubs["external_xpub_3"]}))" +internal_desc="wsh(sortedmulti(2,${xpubs["internal_xpub_1"]},${xpubs["internal_xpub_2"]},${xpubs["internal_xpub_3"]}))" + +external_desc_sum=$(./src/bitcoin-cli -signet getdescriptorinfo $external_desc | jq '.descriptor') +internal_desc_sum=$(./src/bitcoin-cli -signet getdescriptorinfo $internal_desc | jq '.descriptor') + +multisig_ext_desc="{\"desc\": $external_desc_sum, \"active\": true, \"internal\": false, \"timestamp\": \"now\"}" +multisig_int_desc="{\"desc\": $internal_desc_sum, \"active\": true, \"internal\": true, \"timestamp\": \"now\"}" + +multisig_desc="[$multisig_ext_desc, $multisig_int_desc]" +``` + +`external_desc` and `internal_desc` specify the output type (`wsh`, in this case) and the xpubs involved. They also use BIP 67 (`sortedmulti`), so the wallet can be recreated without worrying about the order of xpubs. Conceptually, descriptors describe a list of scriptPubKey (along with information for spending from it) [[source](https://github.com/bitcoin/bitcoin/issues/21199#issuecomment-780772418)]. + +Note that at least two descriptors are usually used, one for internal derivation paths and external ones. There are discussions about eliminating this redundancy, as can been seen in the issue [#17190](https://github.com/bitcoin/bitcoin/issues/17190). + +After creating the descriptors, it is necessary to add the checksum, which is required by the `importdescriptors` RPC. + +The checksum for a descriptor without one can be computed using the `getdescriptorinfo` RPC. The response has the `descriptor` field, which is the descriptor with the checksum added. + +There are other fields that can be added to the descriptors: + +* `active`: Sets the descriptor to be the active one for the corresponding output type (`wsh`, in this case). +* `internal`: Indicates whether matching outputs should be treated as something other than incoming payments (e.g. change). +* `timestamp`: Sets the time from which to start rescanning the blockchain for the descriptor, in UNIX epoch time. + +Documentation for these and other parameters can be found by typing `./src/bitcoin-cli help importdescriptors`. + +`multisig_desc` concatenates external and internal descriptors in a JSON array and then it will be used to create the multisig wallet. + +### 1.3 Create the Multisig Wallet + +To create the multisig wallet, first create an empty one (no keys, HD seed and private keys disabled). + +Then import the descriptors created in the previous step using the `importdescriptors` RPC. + +After that, `getwalletinfo` can be used to check if the wallet was created successfully. + +```bash +./src/bitcoin-cli -signet -named createwallet wallet_name="multisig_wallet_01" disable_private_keys=true blank=true + +./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" importdescriptors "$multisig_desc" + +./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" getwalletinfo +``` + +Once the wallets have already been created and this tutorial needs to be repeated or resumed, it is not necessary to recreate them, just load them with the command below: + +```bash +for ((n=1;n<=3;n++)); do ./src/bitcoin-cli -signet loadwallet "participant_${n}"; done +``` + +### 1.4 Fund the wallet + +The wallet can receive signet coins by generating a new address and passing it as parameters to `getcoins.py` script. + +This script will print a captcha in dot-matrix to the terminal, using unicode Braille characters. After solving the captcha, the coins will be sent directly to the address or wallet (according to the parameters). + +The url used by the script can also be accessed directly. At time of writing, the url is [`https://signetfaucet.com`](https://signetfaucet.com). + +Coins received by the wallet must have at least 1 confirmation before they can be spent. It is necessary to wait for a new block to be mined before continuing. + +```bash +receiving_address=$(./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" getnewaddress) + +./contrib/signet/getcoins.py -c ./src/bitcoin-cli -a $receiving_address +``` + +To copy the receiving address onto the clipboard, use the following command. This can be useful when getting coins via the signet faucet mentioned above. + +```bash +echo -n "$receiving_address" | xclip -sel clip +``` + +The `getbalances` RPC may be used to check the balance. Coins with `trusted` status can be spent. + +```bash +./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" getbalances +``` + +### 1.5 Create a PSBT + +Unlike singlesig wallets, multisig wallets cannot create and sign transactions directly because they require the signatures of the co-signers. Instead they create a Partially Signed Bitcoin Transaction (PSBT). + +PSBT is a data format that allows wallets and other tools to exchange information about a Bitcoin transaction and the signatures necessary to complete it. [[source](https://bitcoinops.org/en/topics/psbt/)] + +The current PSBT version (v0) is defined in [BIP 174](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki). + +For simplicity, the destination address is taken from the `participant_1` wallet in the code above, but it can be any valid bitcoin address. + +The `walletcreatefundedpsbt` RPC is used to create and fund a transaction in the PSBT format. It is the first step in creating the PSBT. + +```bash +balance=$(./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" getbalance) + +amount=$(echo "$balance * 0.8" | bc -l | sed -e 's/^\./0./' -e 's/^-\./-0./') + +destination_addr=$(./src/bitcoin-cli -signet -rpcwallet="participant_1" getnewaddress) + +funded_psbt=$(./src/bitcoin-cli -signet -named -rpcwallet="multisig_wallet_01" walletcreatefundedpsbt outputs="{\"$destination_addr\": $amount}" | jq -r '.psbt') +``` + +There is also the `createpsbt` RPC, which serves the same purpose, but it has no access to the wallet or to the UTXO set. It is functionally the same as `createrawtransaction` and just drops the raw transaction into an otherwise blank PSBT. [[source](https://bitcointalk.org/index.php?topic=5131043.msg50573609#msg50573609)] In most cases, `walletcreatefundedpsbt` solves the problem. + +The `send` RPC can also return a PSBT if more signatures are needed to sign the transaction. + +### 1.6 Decode or Analyze the PSBT + +Optionally, the PSBT can be decoded to a JSON format using `decodepsbt` RPC. + +The `analyzepsbt` RPC analyzes and provides information about the current status of a PSBT and its inputs, e.g. missing signatures. + +```bash +./src/bitcoin-cli -signet decodepsbt $funded_psbt + +./src/bitcoin-cli -signet analyzepsbt $funded_psbt +``` + +### 1.7 Update the PSBT + +In the code above, two PSBTs are created. One signed by `participant_1` wallet and other, by the `participant_2` wallet. + +The `walletprocesspsbt` is used by the wallet to sign a PSBT. + +```bash +psbt_1=$(./src/bitcoin-cli -signet -rpcwallet="participant_1" walletprocesspsbt $funded_psbt | jq '.psbt') + +psbt_2=$(./src/bitcoin-cli -signet -rpcwallet="participant_2" walletprocesspsbt $funded_psbt | jq '.psbt') +``` + +### 1.8 Combine the PSBT + +The PSBT, if signed separately by the co-signers, must be combined into one transaction before being finalized. This is done by `combinepsbt` RPC. + +```bash +combined_psbt=$(./src/bitcoin-cli -signet combinepsbt "[$psbt_1, $psbt_2]") +``` + +There is an RPC called `joinpsbts`, but it has a different purpose than `combinepsbt`. `joinpsbts` joins the inputs from multiple distinct PSBTs into one PSBT. + +In the example above, the PSBTs are the same, but signed by different participants. If the user tries to merge them using `joinpsbts`, the error `Input txid:pos exists in multiple PSBTs` is returned. To be able to merge different PSBTs into one, they must have different inputs and outputs. + +### 1.9 Finalize and Broadcast the PSBT + +The `finalizepsbt` RPC is used to produce a network serialized transaction which can be broadcast with `sendrawtransaction`. + +It checks that all inputs have complete scriptSigs and scriptWitnesses and, if so, encodes them into network serialized transactions. + +```bash +finalized_psbt_hex=$(./src/bitcoin-cli -signet finalizepsbt $combined_psbt | jq -r '.hex') + +./src/bitcoin-cli -signet sendrawtransaction $finalized_psbt_hex +``` + +### 1.10 Alternative Workflow (PSBT sequential signatures) + +Instead of each wallet signing the original PSBT and combining them later, the wallets can also sign the PSBTs sequentially. This is less scalable than the previously presented parallel workflow, but it works. + +After that, the rest of the process is the same: the PSBT is finalized and transmitted to the network. + +```bash +psbt_1=$(./src/bitcoin-cli -signet -rpcwallet="participant_1" walletprocesspsbt $funded_psbt | jq -r '.psbt') + +psbt_2=$(./src/bitcoin-cli -signet -rpcwallet="participant_2" walletprocesspsbt $psbt_1 | jq -r '.psbt') + +finalized_psbt_hex=$(./src/bitcoin-cli -signet finalizepsbt $psbt_2 | jq -r '.hex') + +./src/bitcoin-cli -signet sendrawtransaction $finalized_psbt_hex +``` diff --git a/doc/p2p-bad-ports.md b/doc/p2p-bad-ports.md new file mode 100644 index 000000000000..4f717f97a297 --- /dev/null +++ b/doc/p2p-bad-ports.md @@ -0,0 +1,114 @@ +When Bitcoin Core automatically opens outgoing P2P connections, it chooses +a peer (address and port) from its list of potential peers. This list is +populated with unchecked data gossiped over the P2P network by other peers. + +A malicious actor may gossip an address:port where no Bitcoin node is listening, +or one where a service is listening that is not related to the Bitcoin network. +As a result, this service may occasionally get connection attempts from Bitcoin +nodes. + +"Bad" ports are ones used by services which are usually not open to the public +and usually require authentication. A connection attempt (by Bitcoin Core, +trying to connect because it thinks there is a Bitcoin node on that +address:port) to such service may be considered a malicious action by an +ultra-paranoid administrator. An example for such a port is 22 (ssh). On the +other hand, connection attempts to public services that usually do not require +authentication are unlikely to be considered a malicious action, +e.g. port 80 (http). + +Below is a list of "bad" ports which Bitcoin Core avoids when choosing a peer to +connect to. If a node is listening on such a port, it will likely receive fewer +incoming connections. + + 1: tcpmux + 7: echo + 9: discard + 11: systat + 13: daytime + 15: netstat + 17: qotd + 19: chargen + 20: ftp data + 21: ftp access + 22: ssh + 23: telnet + 25: smtp + 37: time + 42: name + 43: nicname + 53: domain + 69: tftp + 77: priv-rjs + 79: finger + 87: ttylink + 95: supdup + 101: hostname + 102: iso-tsap + 103: gppitnp + 104: acr-nema + 109: pop2 + 110: pop3 + 111: sunrpc + 113: auth + 115: sftp + 117: uucp-path + 119: nntp + 123: NTP + 135: loc-srv /epmap + 137: netbios + 139: netbios + 143: imap2 + 161: snmp + 179: BGP + 389: ldap + 427: SLP (Also used by Apple Filing Protocol) + 465: smtp+ssl + 512: print / exec + 513: login + 514: shell + 515: printer + 526: tempo + 530: courier + 531: chat + 532: netnews + 540: uucp + 548: AFP (Apple Filing Protocol) + 554: rtsp + 556: remotefs + 563: nntp+ssl + 587: smtp (rfc6409) + 601: syslog-conn (rfc3195) + 636: ldap+ssl + 989: ftps-data + 990: ftps + 993: ldap+ssl + 995: pop3+ssl + 1719: h323gatestat + 1720: h323hostcall + 1723: pptp + 2049: nfs + 3659: apple-sasl / PasswordServer + 4045: lockd + 5060: sip + 5061: sips + 6000: X11 + 6566: sane-port + 6665: Alternate IRC + 6666: Alternate IRC + 6667: Standard IRC + 6668: Alternate IRC + 6669: Alternate IRC + 6697: IRC + TLS + 10080: Amanda + +For further information see: + +[pull/23306](https://github.com/bitcoin/bitcoin/pull/23306#issuecomment-947516736) + +[pull/23542](https://github.com/bitcoin/bitcoin/pull/23542) + +[fetch.spec.whatwg.org](https://fetch.spec.whatwg.org/#port-blocking) + +[chromium.googlesource.com](https://chromium.googlesource.com/chromium/src.git/+/refs/heads/main/net/base/port_util.cc) + +[hg.mozilla.org](https://hg.mozilla.org/mozilla-central/file/tip/netwerk/base/nsIOService.cpp) diff --git a/doc/policy/README.md b/doc/policy/README.md new file mode 100644 index 000000000000..27536407e7df --- /dev/null +++ b/doc/policy/README.md @@ -0,0 +1,15 @@ +# Transaction Relay Policy + +**Policy** (Mempool or Transaction Relay Policy) is the node's set of validation rules, in addition +to consensus, enforced for unconfirmed transactions before submitting them to the mempool. These +rules are local to the node and configurable (e.g. `-minrelaytxfee`, `-limitancestorsize`, +`-incrementalrelayfee`). Policy may include restrictions on the transaction itself, the transaction +in relation to the current chain tip, and the transaction in relation to the node's mempool +contents. Policy is *not* applied to transactions in blocks. + +This documentation is not an exhaustive list of all policy rules. + +- [Mempool Limits](mempool-limits.md) +- [Mempool Replacements](mempool-replacements.md) +- [Packages](packages.md) + diff --git a/doc/policy/mempool-limits.md b/doc/policy/mempool-limits.md new file mode 100644 index 000000000000..73ab017f1b16 --- /dev/null +++ b/doc/policy/mempool-limits.md @@ -0,0 +1,65 @@ +# Mempool Limits + +## Definitions + +Given any two transactions Tx0 and Tx1 where Tx1 spends an output of Tx0, +Tx0 is a *parent* of Tx1 and Tx1 is a *child* of Tx0. + +A transaction's *ancestors* include, recursively, its parents, the parents of its parents, etc. +A transaction's *descendants* include, recursively, its children, the children of its children, etc. + +A mempool entry's *ancestor count* is the total number of in-mempool (unconfirmed) transactions in +its ancestor set, including itself. +A mempool entry's *descendant count* is the total number of in-mempool (unconfirmed) transactions in +its descendant set, including itself. + +A mempool entry's *ancestor size* is the aggregated virtual size of in-mempool (unconfirmed) +transactions in its ancestor set, including itself. +A mempool entry's *descendant size* is the aggregated virtual size of in-mempool (unconfirmed) +transactions in its descendant set, including itself. + +Transactions submitted to the mempool must not exceed the ancestor and descendant limits (aka +mempool *package limits*) set by the node (see `-limitancestorcount`, `-limitancestorsize`, +`-limitdescendantcount`, `-limitdescendantsize`). + +## Exemptions + +### CPFP Carve Out + +**CPFP Carve Out** if a transaction candidate for submission to the +mempool would cause some mempool entry to exceed its descendant limits, an exemption is made if all +of the following conditions are met: + +1. The candidate transaction is no more than 10,000 virtual bytes. + +2. The candidate transaction has an ancestor count of 2 (itself and exactly 1 ancestor). + +3. The in-mempool transaction's descendant count, including the candidate transaction, would only + exceed the limit by 1. + +*Rationale*: this rule was introduced to prevent pinning by domination of a transaction's descendant +limits in two-party contract protocols such as LN. Also see the [mailing list +post](https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-November/016518.html). + +This rule was introduced in [PR #15681](https://github.com/bitcoin/bitcoin/pull/15681). + +### Single-Conflict RBF Carve Out + +When a candidate transaction for submission to the mempool would replace mempool entries, it may +also decrease the descendant count of other mempool entries. Since ancestor/descendant limits are +calculated prior to removing the would-be-replaced transactions, they may be overestimated. + +An exemption is given for a candidate transaction that would replace mempool transactions and meets +all of the following conditions: + +1. The candidate transaction has exactly 1 directly conflicting transaction. + +2. The candidate transaction does not spend any unconfirmed inputs that are not also spent by the + directly conflicting transaction. + +The following discounts are given to account for the would-be-replaced transaction(s): + +1. The descendant count limit is temporarily increased by 1. + +2. The descendant size limit temporarily is increased by the virtual size of the to-be-replaced + directly conflicting transaction. diff --git a/doc/policy/mempool-replacements.md b/doc/policy/mempool-replacements.md new file mode 100644 index 000000000000..b3c4239b7312 --- /dev/null +++ b/doc/policy/mempool-replacements.md @@ -0,0 +1,81 @@ +# Mempool Replacements + +## Current Replace-by-Fee Policy + +A transaction conflicts with an in-mempool transaction ("directly conflicting transaction") if they +spend one or more of the same inputs. A transaction may conflict with multiple in-mempool +transactions. + +A transaction ("replacement transaction") may replace its directly conflicting transactions and +their in-mempool descendants (together, "original transactions") if, in addition to passing all +other consensus and policy rules, each of the following conditions are met: + +1. The directly conflicting transactions all signal replaceability explicitly. A transaction is + signaling replaceability if any of its inputs have an nSequence number less than (0xffffffff - 1). + + *Rationale*: See [BIP125 + explanation](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki#motivation). + Use the (`-mempoolfullrbf`) configuration option to allow transaction replacement without enforcement of the + opt-in signaling rule. + +2. The replacement transaction only include an unconfirmed input if that input was included in + one of the directly conflicting transactions. An unconfirmed input spends an output from a + currently-unconfirmed transaction. + + *Rationale*: When RBF was originally implemented, the mempool did not keep track of + ancestor feerates yet. This rule was suggested as a temporary restriction. + +3. The replacement transaction pays an absolute fee of at least the sum paid by the original + transactions. + + *Rationale*: Only requiring the replacement transaction to have a higher feerate could allow an + attacker to bypass node minimum relay feerate requirements and cause the network to repeatedly + relay slightly smaller replacement transactions without adding any more fees. Additionally, if + any of the original transactions would be included in the next block assembled by an economically + rational miner, a replacement policy allowing the replacement transaction to decrease the absolute + fees in the next block would be incentive-incompatible. + +4. The additional fees (difference between absolute fee paid by the replacement transaction and the + sum paid by the original transactions) pays for the replacement transaction's bandwidth at or + above the rate set by the node's incremental relay feerate. For example, if the incremental relay + feerate is 1 satoshi/vB and the replacement transaction is 500 virtual bytes total, then the + replacement pays a fee at least 500 satoshis higher than the sum of the original transactions. + + *Rationale*: Try to prevent DoS attacks where an attacker causes the network to repeatedly relay + transactions each paying a tiny additional amount in fees, e.g. just 1 satoshi. + +5. The number of original transactions does not exceed 100. More precisely, the sum of all + directly conflicting transactions' descendant counts (number of transactions inclusive of itself + and its descendants) must not exceed 100; it is possible that this overestimates the true number + of original transactions. + + *Rationale*: Try to prevent DoS attacks where an attacker is able to easily occupy and flush out + significant portions of the node's mempool using replacements with multiple directly conflicting + transactions, each with large descendant sets. + +6. The replacement transaction's feerate is greater than the feerates of all directly conflicting + transactions. + + *Rationale*: This rule was originally intended to ensure that the replacement transaction is + preferable for block-inclusion, compared to what would be removed from the mempool. This rule + predates ancestor feerate-based transaction selection. + +This set of rules is similar but distinct from BIP125. + +## History + +* Opt-in full replace-by-fee (without inherited signaling) honoured in mempool and mining as of + **v0.12.0** ([PR 6871](https://github.com/bitcoin/bitcoin/pull/6871)). + +* [BIP125](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki) defined based on + Bitcoin Core implementation. + +* The incremental relay feerate used to calculate the required additional fees is distinct from + `-minrelaytxfee` and configurable using `-incrementalrelayfee` + ([PR #9380](https://github.com/bitcoin/bitcoin/pull/9380)). + +* RBF enabled by default in the wallet GUI as of **v0.18.1** ([PR + #11605](https://github.com/bitcoin/bitcoin/pull/11605)). + +* Full replace-by-fee enabled as a configurable mempool policy as of **v24.0** ([PR + #25353](https://github.com/bitcoin/bitcoin/pull/25353)). diff --git a/doc/policy/packages.md b/doc/policy/packages.md new file mode 100644 index 000000000000..274854ddf93d --- /dev/null +++ b/doc/policy/packages.md @@ -0,0 +1,119 @@ +# Package Mempool Accept + +## Definitions + +A **package** is an ordered list of transactions, representable by a connected Directed Acyclic +Graph (a directed edge exists between a transaction that spends the output of another transaction). + +For every transaction `t` in a **topologically sorted** package, if any of its parents are present +in the package, they appear somewhere in the list before `t`. + +A **child-with-unconfirmed-parents** package is a topologically sorted package that consists of +exactly one child and all of its unconfirmed parents (no other transactions may be present). +The last transaction in the package is the child, and its package can be canonically defined based +on the current state: each of its inputs must be available in the UTXO set as of the current chain +tip or some preceding transaction in the package. + +## Package Mempool Acceptance Rules + +The following rules are enforced for all packages: + +* Packages cannot exceed `MAX_PACKAGE_COUNT=25` count and `MAX_PACKAGE_SIZE=101KvB` total size + (#20833) + + - *Rationale*: This is already enforced as mempool ancestor/descendant limits. If + transactions in a package are all related, exceeding this limit would mean that the package + can either be split up or it wouldn't pass individual mempool policy. + + - Note that, if these mempool limits change, package limits should be reconsidered. Users may + also configure their mempool limits differently. + +* Packages must be topologically sorted. (#20833) + +* Packages cannot have conflicting transactions, i.e. no two transactions in a package can spend + the same inputs. Packages cannot have duplicate transactions. (#20833) + +* No transaction in a package can conflict with a mempool transaction. Replace By Fee is + currently disabled for packages. (#20833) + + - Package RBF may be enabled in the future. + +* When packages are evaluated against ancestor/descendant limits, the union of all transactions' + descendants and ancestors is considered. (#21800) + + - *Rationale*: This is essentially a "worst case" heuristic intended for packages that are + heavily connected, i.e. some transaction in the package is the ancestor or descendant of all + the other transactions. + +The following rules are only enforced for packages to be submitted to the mempool (not enforced for +test accepts): + +* Packages must be child-with-unconfirmed-parents packages. This also means packages must contain at + least 2 transactions. (#22674) + + - *Rationale*: This allows for fee-bumping by CPFP. Allowing multiple parents makes it possible + to fee-bump a batch of transactions. Restricting packages to a defined topology is easier to + reason about and simplifies the validation logic greatly. + + - Warning: Batched fee-bumping may be unsafe for some use cases. Users and application developers + should take caution if utilizing multi-parent packages. + +* Transactions in the package that have the same txid as another transaction already in the mempool + will be removed from the package prior to submission ("deduplication"). + + - *Rationale*: Node operators are free to set their mempool policies however they please, nodes + may receive transactions in different orders, and malicious counterparties may try to take + advantage of policy differences to pin or delay propagation of transactions. As such, it's + possible for some package transaction(s) to already be in the mempool, and there is no need to + repeat validation for those transactions or double-count them in fees. + + - *Rationale*: We want to prevent potential censorship vectors. We should not reject entire + packages because we already have one of the transactions. Also, if an attacker first broadcasts + a competing package or transaction with a mutated witness, even though the two + same-txid-different-witness transactions are conflicting and cannot replace each other, the + honest package should still be considered for acceptance. + +### Package Fees and Feerate + +*Package Feerate* is the total modified fees (base fees + any fee delta from +`prioritisetransaction`) divided by the total virtual size of all transactions in the package. +If any transactions in the package are already in the mempool, they are not submitted again +("deduplicated") and are thus excluded from this calculation. + +To meet the two feerate requirements of a mempool, i.e., the pre-configured minimum relay feerate +(`-minrelaytxfee`) and the dynamic mempool minimum feerate, the total package feerate is used instead +of the individual feerate. The individual transactions are allowed to be below the feerate +requirements if the package meets the feerate requirements. For example, the parent(s) in the +package can pay no fees but be paid for by the child. + +*Rationale*: This can be thought of as "CPFP within a package," solving the issue of a parent not +meeting minimum fees on its own. This would allow contracting applications to adjust their fees at +broadcast time instead of overshooting or risking becoming stuck or pinned. + +*Rationale*: It would be incorrect to use the fees of transactions that are already in the mempool, as +we do not want a transaction's fees to be double-counted. + +Implementation Note: Transactions within a package are always validated individually first, and +package validation is used for the transactions that failed. Since package feerate is only +calculated using transactions that are not in the mempool, this implementation detail affects the +outcome of package validation. + +*Rationale*: Packages are intended for incentive-compatible fee-bumping: transaction B is a +"legitimate" fee-bump for transaction A only if B is a descendant of A and has a *higher* feerate +than A. We want to prevent "parents pay for children" behavior; fees of parents should not help +their children, since the parents can be mined without the child. More generally, if transaction A +is not needed in order for transaction B to be mined, A's fees cannot help B. In a +child-with-parents package, simply excluding any parent transactions that meet feerate requirements +individually is sufficient to ensure this. + +*Rationale*: We must not allow a low-feerate child to prevent its parent from being accepted; fees +of children should not negatively impact their parents, since they are not necessary for the parents +to be mined. More generally, if transaction B is not needed in order for transaction A to be mined, +B's fees cannot harm A. In a child-with-parents package, simply validating parents individually +first is sufficient to ensure this. + +*Rationale*: As a principle, we want to avoid accidentally restricting policy in order to be +backward-compatible for users and applications that rely on p2p transaction relay. Concretely, +package validation should not prevent the acceptance of a transaction that would otherwise be +policy-valid on its own. By always accepting a transaction that passes individual validation before +trying package validation, we prevent any unintentional restriction of policy. diff --git a/doc/productivity.md b/doc/productivity.md index a01c6f545db8..e9b7bc270c67 100644 --- a/doc/productivity.md +++ b/doc/productivity.md @@ -9,6 +9,7 @@ Table of Contents * [Disable features with `./configure`](#disable-features-with-configure) * [Make use of your threads with `make -j`](#make-use-of-your-threads-with-make--j) * [Only build what you need](#only-build-what-you-need) + * [Compile on multiple machines](#compile-on-multiple-machines) * [Multiple working directories with `git worktrees`](#multiple-working-directories-with-git-worktrees) * [Interactive "dummy rebases" for fixups and execs with `git merge-base`](#interactive-dummy-rebases-for-fixups-and-execs-with-git-merge-base) * [Writing code](#writing-code) @@ -81,6 +82,10 @@ make -C src bitcoin_bench (You can and should combine this with `-j`, as above, for a parallel build.) +### Compile on multiple machines + +If you have more than one computer at your disposal, you can use [distcc](https://www.distcc.org) to speed up compilation. This is easiest when all computers run the same operating system and compiler version. + ### Multiple working directories with `git worktrees` If you work with multiple branches or multiple copies of the repository, you should try `git worktrees`. diff --git a/doc/psbt.md b/doc/psbt.md index c411b31d5d96..0f31cb8eba63 100644 --- a/doc/psbt.md +++ b/doc/psbt.md @@ -92,6 +92,9 @@ hardware implementations will typically implement multiple roles simultaneously. #### Multisig with multiple Bitcoin Core instances +For a quick start see [Basic M-of-N multisig example using descriptor wallets and PSBTs](./descriptors.md#basic-multisig-example). +If you are using legacy wallets feel free to continue with the example provided here. + Alice, Bob, and Carol want to create a 2-of-3 multisig address. They're all using Bitcoin Core. We assume their wallets only contain the multisig funds. In case they also have a personal wallet, this can be accomplished through the diff --git a/doc/release-notes-19762.md b/doc/release-notes-19762.md new file mode 100644 index 000000000000..4dc45fb2c86b --- /dev/null +++ b/doc/release-notes-19762.md @@ -0,0 +1,19 @@ +JSON-RPC +--- + +All JSON-RPC methods accept a new [named +parameter](JSON-RPC-interface.md#parameter-passing) called `args` that can +contain positional parameter values. This is a convenience to allow some +parameter values to be passed by name without having to name every value. The +python test framework and `bitcoin-cli` tool both take advantage of this, so +for example: + +```sh +bitcoin-cli -named createwallet wallet_name=mywallet load_on_startup=1 +``` + +Can now be shortened to: + +```sh +bitcoin-cli -named createwallet mywallet load_on_startup=1 +``` diff --git a/doc/release-notes-22087.md b/doc/release-notes-22087.md new file mode 100644 index 000000000000..8d7fd242b270 --- /dev/null +++ b/doc/release-notes-22087.md @@ -0,0 +1,4 @@ +Updated settings +---------------- + +- Ports specified in `-port` and `-rpcport` options are now validated at startup. Values that previously worked and were considered valid can now result in errors. (#22087) diff --git a/doc/release-notes-25412.md b/doc/release-notes-25412.md new file mode 100644 index 000000000000..b11fe73d450a --- /dev/null +++ b/doc/release-notes-25412.md @@ -0,0 +1,5 @@ +New REST endpoint +----------------- + +- A new `/rest/deploymentinfo` endpoint has been added for fetching various + state info regarding deployments of consensus changes. (#25412) diff --git a/doc/release-notes-25730.md b/doc/release-notes-25730.md new file mode 100644 index 000000000000..33393cf31490 --- /dev/null +++ b/doc/release-notes-25730.md @@ -0,0 +1,6 @@ +RPC Wallet +---------- + +- RPC `listunspent` now has a new argument `include_immature_coinbase` + to include coinbase UTXOs that don't meet the minimum spendability + depth requirement (which before were silently skipped). (#25730) \ No newline at end of file diff --git a/doc/release-notes-25934.md b/doc/release-notes-25934.md new file mode 100644 index 000000000000..b4f1ae0d3cf7 --- /dev/null +++ b/doc/release-notes-25934.md @@ -0,0 +1,8 @@ +Low-level changes +================= + +RPC +--- + +- RPC `listsinceblock` now accepts an optional `label` argument + to fetch incoming transactions having the specified label. (#25934) \ No newline at end of file diff --git a/doc/release-notes-26213.md b/doc/release-notes-26213.md new file mode 100644 index 000000000000..e78d718ca998 --- /dev/null +++ b/doc/release-notes-26213.md @@ -0,0 +1,8 @@ +Low-level changes +================= + +- Previously `setban`, `addpeeraddress`, `walletcreatefundedpsbt`, methods + allowed non-boolean and non-null values to be passed as boolean parameters. + Any string, number, array, or object value that was passed would be treated + as false. After this change, passing any value except `true`, `false`, or + `null` now triggers a JSON value is not of expected type error. (#26213) diff --git a/doc/release-notes-26265.md b/doc/release-notes-26265.md new file mode 100644 index 000000000000..ca2313d95694 --- /dev/null +++ b/doc/release-notes-26265.md @@ -0,0 +1,6 @@ +P2P and network changes +--------- + +- Transactions of non-witness size 65 and above are now allowed by mempool + and relay policy. This is to better reflect the actual afforded protections + against CVE-2017-12842 and open up additional use-cases of smaller transaction sizes. (#26265) diff --git a/doc/release-notes-26628.md b/doc/release-notes-26628.md new file mode 100644 index 000000000000..48a07c1e8187 --- /dev/null +++ b/doc/release-notes-26628.md @@ -0,0 +1,4 @@ +JSON-RPC +--- + +The JSON-RPC server now rejects requests where a parameter is specified multiple times with the same name, instead of silently overwriting earlier parameter values with later ones. (#26628) diff --git a/doc/release-notes-empty-template.md b/doc/release-notes-empty-template.md new file mode 100644 index 000000000000..4cd2314308d0 --- /dev/null +++ b/doc/release-notes-empty-template.md @@ -0,0 +1,99 @@ +*The release notes draft is a temporary file that can be added to by anyone. See +[/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) +for the process.* + +*version* Release Notes Draft +=============================== + +Bitcoin Core version *version* is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +Updated RPCs +------------ + + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +Build System +------------ + +Updated settings +---------------- + + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings +------------ + +Tools and Utilities +------------------- + +Wallet +------ + +GUI changes +----------- + +Low-level changes +================= + +RPC +--- + +Tests +----- + +*version* change log +==================== + +Credits +======= + +Thanks to everyone who directly contributed to this release: + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes.md b/doc/release-notes.md deleted file mode 100644 index dc28ccb9edd5..000000000000 --- a/doc/release-notes.md +++ /dev/null @@ -1,223 +0,0 @@ -# Release notes now being edited on https://github.com/bitcoin-core/bitcoin-devwiki/wiki/22.0-Release-Notes-draft - -*After branching off for a major version release of Bitcoin Core, use this -template to create the initial release notes draft.* - -*The release notes draft is a temporary file that can be added to by anyone. See -[/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) -for the process.* - -*Create the draft, named* "*version* Release Notes Draft" -*(e.g. "0.20.0 Release Notes Draft"), as a collaborative wiki in:* - -https://github.com/bitcoin-core/bitcoin-devwiki/wiki/ - -*Before the final release, move the notes back to this git repository.* - -*version* Release Notes Draft -=============================== - -Bitcoin Core version *version* is now available from: - - - -This release includes new features, various bug fixes and performance -improvements, as well as updated translations. - -Please report bugs using the issue tracker at GitHub: - - - -To receive security and update notifications, please subscribe to: - - - -How to Upgrade -============== - -If you are running an older version, shut it down. Wait until it has completely -shut down (which might take a few minutes in some cases), then run the -installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) -or `bitcoind`/`bitcoin-qt` (on Linux). - -Upgrading directly from a version of Bitcoin Core that has reached its EOL is -possible, but it might take some time if the data directory needs to be migrated. Old -wallet versions of Bitcoin Core are generally supported. - -Compatibility -============== - -Bitcoin Core is supported and extensively tested on operating systems -using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin -Core should also work on most other Unix-like systems but is not as -frequently tested on them. It is not recommended to use Bitcoin Core on -unsupported systems. - -From Bitcoin Core 22.0 onwards, macOS versions earlier than 10.14 are no longer supported. - -Notable changes -=============== - -P2P and network changes ------------------------ - -- This release removes support for Tor version 2 hidden services in favor of Tor - v3 only, as the Tor network [dropped support for Tor - v2](https://blog.torproject.org/v2-deprecation-timeline) with the release of - Tor version 0.4.6. Henceforth, Bitcoin Core ignores Tor v2 addresses; it - neither rumors them over the network to other peers, nor stores them in memory - or to `peers.dat`. (#22050) - -- Added NAT-PMP port mapping support via - [`libnatpmp`](https://miniupnp.tuxfamily.org/libnatpmp.html). (#18077) - -Updated RPCs ------------- - -- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) - being implemented, behavior for all RPCs that accept addresses is changed when - a native witness version 1 (or higher) is passed. These now require a Bech32m - encoding instead of a Bech32 one, and Bech32m encoding will be used for such - addresses in RPC output as well. No version 1 addresses should be created - for mainnet until consensus rules are adopted that give them meaning - (e.g. through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). - Once that happens, Bech32m is expected to be used for them, so this shouldn't - affect any production systems, but may be observed on other networks where such - addresses already have meaning (like signet). (#20861) - -- The `getpeerinfo` RPC returns two new boolean fields, `bip152_hb_to` and - `bip152_hb_from`, that respectively indicate whether we selected a peer to be - in compact blocks high-bandwidth mode or whether a peer selected us as a - compact blocks high-bandwidth peer. High-bandwidth peers send new block - announcements via a `cmpctblock` message rather than the usual inv/headers - announcements. See BIP 152 for more details. (#19776) - -- `getpeerinfo` no longer returns the following fields: `addnode`, `banscore`, - and `whitelisted`, which were previously deprecated in 0.21. Instead of - `addnode`, the `connection_type` field returns manual. Instead of - `whitelisted`, the `permissions` field indicates if the peer has special - privileges. The `banscore` field has simply been removed. (#20755) - -- The following RPCs: `gettxout`, `getrawtransaction`, `decoderawtransaction`, - `decodescript`, `gettransaction`, and REST endpoints: `/rest/tx`, - `/rest/getutxos`, `/rest/block` deprecated the following fields (which are no - longer returned in the responses by default): `addresses`, `reqSigs`. - The `-deprecatedrpc=addresses` flag must be passed for these fields to be - included in the RPC response. This flag/option will be available only for this major release, after which - the deprecation will be removed entirely. Note that these fields are attributes of - the `scriptPubKey` object returned in the RPC response. However, in the response - of `decodescript` these fields are top-level attributes, and included again as attributes - of the `scriptPubKey` object. (#20286) - -- When creating a hex-encoded bitcoin transaction using the `bitcoin-tx` utility - with the `-json` option set, the following fields: `addresses`, `reqSigs` are no longer - returned in the tx output of the response. (#20286) - -- The `listbanned` RPC now returns two new numeric fields: `ban_duration` and `time_remaining`. - Respectively, these new fields indicate the duration of a ban and the time remaining until a ban expires, - both in seconds. Additionally, the `ban_created` field is repositioned to come before `banned_until`. (#21602) - -- The `getnodeaddresses` RPC now returns a "network" field indicating the - network type (ipv4, ipv6, onion, or i2p) for each address. (#21594) - -- `getnodeaddresses` now also accepts a "network" argument (ipv4, ipv6, onion, - or i2p) to return only addresses of the specified network. (#21843) - -- The `testmempoolaccept` RPC now accepts multiple transactions (still experimental at the moment, - API may be unstable). This is intended for testing transaction packages with dependency - relationships; it is not recommended for batch-validating independent transactions. In addition to - mempool policy, package policies apply: the list cannot contain more than 25 transactions or have a - total size exceeding 101K virtual bytes, and cannot conflict with (spend the same inputs as) each other or - the mempool, even if it would be a valid BIP125 replace-by-fee. There are some known limitations to - the accuracy of the test accept: it's possible for `testmempoolaccept` to return "allowed"=True for a - group of transactions, but "too-long-mempool-chain" if they are actually submitted. (#20833) - -- `addmultisigaddress` and `createmultisig` now support up to 20 keys for - Segwit addresses. (#20867) - -Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. - -New RPCs --------- - -Build System ------------- - -New settings ------------- - -- The `-natpmp` option has been added to use NAT-PMP to map the listening port. - If both UPnP and NAT-PMP are enabled, a successful allocation from UPnP - prevails over one from NAT-PMP. (#18077) - -Updated settings ----------------- - -Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. - -- Passing an invalid `-rpcauth` argument now cause bitcoind to fail to start. (#20461) - -Tools and Utilities -------------------- - -- A new CLI `-addrinfo` command returns the number of addresses known to the - node per network type (including Tor v2 versus v3) and total. This can be - useful to see if the node knows enough addresses in a network to use options - like `-onlynet=` or to upgrade to this release of Bitcoin Core 22.0 - that supports Tor v3 only. (#21595) - -- A new `-rpcwaittimeout` argument to `bitcoin-cli` sets the timeout - in seconds to use with `-rpcwait`. If the timeout expires, - `bitcoin-cli` will report a failure. (#21056) - -Wallet ------- - -- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. - The RPC returns public versions of all imported descriptors, including their timestamp and flags. - For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) - -- The `bumpfee` RPC is not available with wallets that have private keys - disabled. `psbtbumpfee` can be used instead. (#20891) - -- The `fundrawtransaction`, `send` and `walletcreatefundedpsbt` RPCs now support an `include_unsafe` option - that when `true` allows using unsafe inputs to fund the transaction. - Note that the resulting transaction may become invalid if one of the unsafe inputs disappears. - If that happens, the transaction must be funded with different inputs and republished. (#21359) - -- We now support up to 20 keys in `multi()` and `sortedmulti()` descriptors - under `wsh()`. (#20867) - -GUI changes ------------ - -Low-level changes -================= - -RPC ---- - -- The RPC server can process a limited number of simultaneous RPC requests. - Previously, if this limit was exceeded, the RPC server would respond with - [status code 500 (`HTTP_INTERNAL_SERVER_ERROR`)](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#5xx_server_errors). - Now it returns status code 503 (`HTTP_SERVICE_UNAVAILABLE`). (#18335) - -- Error codes have been updated to be more accurate for the following error cases (#18466): - - `signmessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the - passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). - - `verifymessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the - passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). - - `verifymessage` now returns RPC_TYPE_ERROR (-3) if the passed signature - is malformed. Previously returned RPC_INVALID_ADDRESS_OR_KEY (-5). - -Tests ------ - -Credits -======= - -Thanks to everyone who directly contributed to this release: - - -As well as to everyone that helped with translations on -[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-0.20.2.md b/doc/release-notes/release-notes-0.20.2.md new file mode 100644 index 000000000000..ad001bc9c12c --- /dev/null +++ b/doc/release-notes/release-notes-0.20.2.md @@ -0,0 +1,165 @@ +0.20.2 Release Notes +==================== + +Bitcoin Core version 0.20.2 is now available from: + + + +This minor release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +Known Bugs +========== + +The process for generating the source code release ("tarball") has changed in an +effort to make it more complete, however, there are a few regressions in +this release: + +- The generated `configure` script is currently missing, and you will need to + install autotools and run `./autogen.sh` before you can run + `./configure`. This is the same as when checking out from git. + +- Instead of running `make` simply, you should instead run + `BITCOIN_GENBUILD_NO_GIT=1 make`. + +Notable changes +=============== + +Changes regarding misbehaving peers +----------------------------------- + +Peers that misbehave (e.g. send us invalid blocks) are now referred to as +discouraged nodes in log output, as they're not (and weren't) strictly banned: +incoming connections are still allowed from them, but they're preferred for +eviction. + +Furthermore, a few additional changes are introduced to how discouraged +addresses are treated: + +- Discouraging an address does not time out automatically after 24 hours + (or the `-bantime` setting). Depending on traffic from other peers, + discouragement may time out at an indeterminate time. + +- Discouragement is not persisted over restarts. + +- There is no method to list discouraged addresses. They are not returned by + the `listbanned` RPC. That RPC also no longer reports the `ban_reason` + field, as `"manually added"` is the only remaining option. + +- Discouragement cannot be removed with the `setban remove` RPC command. + If you need to remove a discouragement, you can remove all discouragements by + stop-starting your node. + +Notification changes +-------------------- + +`-walletnotify` notifications are now sent for wallet transactions that are +removed from the mempool because they conflict with a new block. These +notifications were sent previously before the v0.19 release, but had been +broken since that release (bug +[#18325](https://github.com/bitcoin/bitcoin/issues/18325)). + +PSBT changes +------------ + +PSBTs will contain both the non-witness utxo and the witness utxo for segwit +inputs in order to restore compatibility with wallet software that are now +requiring the full previous transaction for segwit inputs. The witness utxo +is still provided to maintain compatibility with software which relied on its +existence to determine whether an input was segwit. + +0.20.2 change log +================= + +### P2P protocol and network code + +- #19620 Add txids with non-standard inputs to reject filter (sdaftuar) +- #20146 Send post-verack handshake messages at most once (MarcoFalke) + +### Wallet + +- #19740 Simplify and fix CWallet::SignTransaction (achow101) + +### RPC and other APIs + +- #19836 Properly deserialize txs with witness before signing (MarcoFalke) +- #20731 Add missing description of vout in getrawtransaction help text (benthecarman) + +### Build system + +- #20142 build: set minimum required Boost to 1.48.0 (fanquake) +- #20298 use the new plistlib API (jonasschnelli) +- #20880 gitian: Use custom MacOS code signing tool (achow101) +- #22190 Use latest signapple commit (achow101) + +### Tests and QA + +- #19839 Set appveyor vm version to previous Visual Studio 2019 release. (sipsorcery) +- #19842 Update the vcpkg checkout commit ID in appveyor config. (sipsorcery) +- #20562 Test that a fully signed tx given to signrawtx is unchanged (achow101) + +### Miscellaneous + +- #19192 Extract net permissions doc (MarcoFalke) +- #19777 Correct description for getblockstats's txs field (shesek) +- #20080 Strip any trailing / in -datadir and -blocksdir paths (hebasto) +- #20082 fixes read buffer to use min rather than max (EthanHeilman) +- #20141 Avoid the use of abs64 in timedata (sipa) +- #20756 Add missing field (permissions) to the getpeerinfo help (amitiuttarwar) +- #20861 BIP 350: Implement Bech32m and use it for v1+ segwit addresses (sipa) +- #22124 Update translations after closing 0.20.x on Transifex (hebasto) +- #21471 fix bech32_encode calls in gen_key_io_test_vectors.py (sipa) +- #22837 mention bech32m/BIP350 in doc/descriptors.md (sipa) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Aaron Clauson +- Amiti Uttarwar +- Andrew Chow +- Ethan Heilman +- fanquake +- Hennadii Stepanov +- Jonas Schnelli +- MarcoFalke +- Nadav Ivgi +- Pieter Wuille +- Suhas Daftuar + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-0.21.2.md b/doc/release-notes/release-notes-0.21.2.md new file mode 100644 index 000000000000..3b33c48a2698 --- /dev/null +++ b/doc/release-notes/release-notes-0.21.2.md @@ -0,0 +1,109 @@ +0.21.2 Release Notes +==================== + +Bitcoin Core version 0.21.2 is now available from: + + + +This minor release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + + +0.21.2 change log +================= + +### P2P protocol and network code + +- #21644 use NetPermissions::HasFlag() in CConnman::Bind() (jonatack) +- #22569 Rate limit the processing of rumoured addresses (sipa) + +### Wallet + +- #21907 Do not iterate a directory if having an error while accessing it (hebasto) + +### RPC + +- #19361 Reset scantxoutset progress before inferring descriptors (prusnak) + +### Build System + +- #21932 depends: update Qt 5.9 source url (kittywhiskers) +- #22017 Update Windows code signing certificate (achow101) +- #22191 Use custom MacOS code signing tool (achow101) +- #22713 Fix build with Boost 1.77.0 (sizeofvoid) + +### Tests and QA + +- #20182 Build with --enable-werror by default, and document exceptions (hebasto) +- #20535 Fix intermittent feature_taproot issue (MarcoFalke) +- #21663 Fix macOS brew install command (hebasto) +- #22279 add missing ECCVerifyHandle to base_encode_decode (apoelstra) +- #22730 Run fuzzer task for the master branch only (hebasto) + +### GUI + +- #277 Do not use QClipboard::Selection on Windows and macOS. (hebasto) +- #280 Remove user input from URI error message (prayank23) +- #365 Draw "eye" sign at the beginning of watch-only addresses (hebasto) + +### Miscellaneous + +- #22002 Fix crash when parsing command line with -noincludeconf=0 (MarcoFalke) +- #22137 util: Properly handle -noincludeconf on command line (take 2) (MarcoFalke) + + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- Andrew Poelstra +- fanquake +- Hennadii Stepanov +- Jon Atack +- Kittywhiskers Van Gogh +- Luke Dashjr +- MarcoFalke +- Pavol Rusnak +- Pieter Wuille +- prayank23 +- Rafael Sadowski +- W. J. van der Laan + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-22.0.md b/doc/release-notes/release-notes-22.0.md new file mode 100644 index 000000000000..972c91aa6faf --- /dev/null +++ b/doc/release-notes/release-notes-22.0.md @@ -0,0 +1,1163 @@ +22.0 Release Notes +================== + +Bitcoin Core version 22.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 22.0 onwards, macOS versions earlier than 10.14 are no longer supported. + +Notable changes +=============== + +P2P and network changes +----------------------- +- Added support for running Bitcoin Core as an + [I2P (Invisible Internet Project)](https://en.wikipedia.org/wiki/I2P) service + and connect to such services. See [i2p.md](https://github.com/bitcoin/bitcoin/blob/22.x/doc/i2p.md) for details. (#20685) +- This release removes support for Tor version 2 hidden services in favor of Tor + v3 only, as the Tor network [dropped support for Tor + v2](https://blog.torproject.org/v2-deprecation-timeline) with the release of + Tor version 0.4.6. Henceforth, Bitcoin Core ignores Tor v2 addresses; it + neither rumors them over the network to other peers, nor stores them in memory + or to `peers.dat`. (#22050) + +- Added NAT-PMP port mapping support via + [`libnatpmp`](https://miniupnp.tuxfamily.org/libnatpmp.html). (#18077) + +New and Updated RPCs +-------------------- + +- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) + being implemented, behavior for all RPCs that accept addresses is changed when + a native witness version 1 (or higher) is passed. These now require a Bech32m + encoding instead of a Bech32 one, and Bech32m encoding will be used for such + addresses in RPC output as well. No version 1 addresses should be created + for mainnet until consensus rules are adopted that give them meaning + (as will happen through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). + Once that happens, Bech32m is expected to be used for them, so this shouldn't + affect any production systems, but may be observed on other networks where such + addresses already have meaning (like signet). (#20861) + +- The `getpeerinfo` RPC returns two new boolean fields, `bip152_hb_to` and + `bip152_hb_from`, that respectively indicate whether we selected a peer to be + in compact blocks high-bandwidth mode or whether a peer selected us as a + compact blocks high-bandwidth peer. High-bandwidth peers send new block + announcements via a `cmpctblock` message rather than the usual inv/headers + announcements. See BIP 152 for more details. (#19776) + +- `getpeerinfo` no longer returns the following fields: `addnode`, `banscore`, + and `whitelisted`, which were previously deprecated in 0.21. Instead of + `addnode`, the `connection_type` field returns manual. Instead of + `whitelisted`, the `permissions` field indicates if the peer has special + privileges. The `banscore` field has simply been removed. (#20755) + +- The following RPCs: `gettxout`, `getrawtransaction`, `decoderawtransaction`, + `decodescript`, `gettransaction`, and REST endpoints: `/rest/tx`, + `/rest/getutxos`, `/rest/block` deprecated the following fields (which are no + longer returned in the responses by default): `addresses`, `reqSigs`. + The `-deprecatedrpc=addresses` flag must be passed for these fields to be + included in the RPC response. This flag/option will be available only for this major release, after which + the deprecation will be removed entirely. Note that these fields are attributes of + the `scriptPubKey` object returned in the RPC response. However, in the response + of `decodescript` these fields are top-level attributes, and included again as attributes + of the `scriptPubKey` object. (#20286) + +- When creating a hex-encoded bitcoin transaction using the `bitcoin-tx` utility + with the `-json` option set, the following fields: `addresses`, `reqSigs` are no longer + returned in the tx output of the response. (#20286) + +- The `listbanned` RPC now returns two new numeric fields: `ban_duration` and `time_remaining`. + Respectively, these new fields indicate the duration of a ban and the time remaining until a ban expires, + both in seconds. Additionally, the `ban_created` field is repositioned to come before `banned_until`. (#21602) + +- The `setban` RPC can ban onion addresses again. This fixes a regression introduced in version 0.21.0. (#20852) + +- The `getnodeaddresses` RPC now returns a "network" field indicating the + network type (ipv4, ipv6, onion, or i2p) for each address. (#21594) + +- `getnodeaddresses` now also accepts a "network" argument (ipv4, ipv6, onion, + or i2p) to return only addresses of the specified network. (#21843) + +- The `testmempoolaccept` RPC now accepts multiple transactions (still experimental at the moment, + API may be unstable). This is intended for testing transaction packages with dependency + relationships; it is not recommended for batch-validating independent transactions. In addition to + mempool policy, package policies apply: the list cannot contain more than 25 transactions or have a + total size exceeding 101K virtual bytes, and cannot conflict with (spend the same inputs as) each other or + the mempool, even if it would be a valid BIP125 replace-by-fee. There are some known limitations to + the accuracy of the test accept: it's possible for `testmempoolaccept` to return "allowed"=True for a + group of transactions, but "too-long-mempool-chain" if they are actually submitted. (#20833) + +- `addmultisigaddress` and `createmultisig` now support up to 20 keys for + Segwit addresses. (#20867) + +Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. + +Build System +------------ + +- Release binaries are now produced using the new `guix`-based build system. + The [/doc/release-process.md](/doc/release-process.md) document has been updated accordingly. + +Files +----- + +- The list of banned hosts and networks (via `setban` RPC) is now saved on disk + in JSON format in `banlist.json` instead of `banlist.dat`. `banlist.dat` is + only read on startup if `banlist.json` is not present. Changes are only written to the new + `banlist.json`. A future version of Bitcoin Core may completely ignore + `banlist.dat`. (#20966) + +New settings +------------ + +- The `-natpmp` option has been added to use NAT-PMP to map the listening port. + If both UPnP and NAT-PMP are enabled, a successful allocation from UPnP + prevails over one from NAT-PMP. (#18077) + +Updated settings +---------------- + +Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. + +- Passing an invalid `-rpcauth` argument now cause bitcoind to fail to start. (#20461) + +Tools and Utilities +------------------- + +- A new CLI `-addrinfo` command returns the number of addresses known to the + node per network type (including Tor v2 versus v3) and total. This can be + useful to see if the node knows enough addresses in a network to use options + like `-onlynet=` or to upgrade to this release of Bitcoin Core 22.0 + that supports Tor v3 only. (#21595) + +- A new `-rpcwaittimeout` argument to `bitcoin-cli` sets the timeout + in seconds to use with `-rpcwait`. If the timeout expires, + `bitcoin-cli` will report a failure. (#21056) + +Wallet +------ + +- External signers such as hardware wallets can now be used through the new RPC methods `enumeratesigners` and `displayaddress`. Support is also added to the `send` RPC call. This feature is experimental. See [external-signer.md](https://github.com/bitcoin/bitcoin/blob/22.x/doc/external-signer.md) for details. (#16546) + +- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. + The RPC returns public versions of all imported descriptors, including their timestamp and flags. + For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) + +- The `bumpfee` RPC is not available with wallets that have private keys + disabled. `psbtbumpfee` can be used instead. (#20891) + +- The `fundrawtransaction`, `send` and `walletcreatefundedpsbt` RPCs now support an `include_unsafe` option + that when `true` allows using unsafe inputs to fund the transaction. + Note that the resulting transaction may become invalid if one of the unsafe inputs disappears. + If that happens, the transaction must be funded with different inputs and republished. (#21359) + +- We now support up to 20 keys in `multi()` and `sortedmulti()` descriptors + under `wsh()`. (#20867) + +- Taproot descriptors can be imported into the wallet only after activation has occurred on the network (e.g. mainnet, testnet, signet) in use. See [descriptors.md](https://github.com/bitcoin/bitcoin/blob/22.x/doc/descriptors.md) for supported descriptors. + +GUI changes +----------- + +- External signers such as hardware wallets can now be used. These require an external tool such as [HWI](https://github.com/bitcoin-core/HWI) to be installed and configured under Options -> Wallet. When creating a new wallet a new option "External signer" will appear in the dialog. If the device is detected, its name is suggested as the wallet name. The watch-only keys are then automatically imported. Receive addresses can be verified on the device. The send dialog will automatically use the connected device. This feature is experimental and the UI may freeze for a few seconds when performing these actions. + +Low-level changes +================= + +RPC +--- + +- The RPC server can process a limited number of simultaneous RPC requests. + Previously, if this limit was exceeded, the RPC server would respond with + [status code 500 (`HTTP_INTERNAL_SERVER_ERROR`)](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#5xx_server_errors). + Now it returns status code 503 (`HTTP_SERVICE_UNAVAILABLE`). (#18335) + +- Error codes have been updated to be more accurate for the following error cases (#18466): + - `signmessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_TYPE_ERROR (-3) if the passed signature + is malformed. Previously returned RPC_INVALID_ADDRESS_OR_KEY (-5). + +Tests +----- + +22.0 change log +=============== + +A detailed list of changes in this version follows. To keep the list to a manageable length, small refactors and typo fixes are not included, and similar changes are sometimes condensed into one line. + +### Consensus +- bitcoin/bitcoin#19438 Introduce deploymentstatus (ajtowns) +- bitcoin/bitcoin#20207 Follow-up extra comments on taproot code and tests (sipa) +- bitcoin/bitcoin#21330 Deal with missing data in signature hashes more consistently (sipa) + +### Policy +- bitcoin/bitcoin#18766 Disable fee estimation in blocksonly mode (by removing the fee estimates global) (darosior) +- bitcoin/bitcoin#20497 Add `MAX_STANDARD_SCRIPTSIG_SIZE` to policy (sanket1729) +- bitcoin/bitcoin#20611 Move `TX_MAX_STANDARD_VERSION` to policy (MarcoFalke) + +### Mining +- bitcoin/bitcoin#19937, bitcoin/bitcoin#20923 Signet mining utility (ajtowns) + +### Block and transaction handling +- bitcoin/bitcoin#14501 Fix possible data race when committing block files (luke-jr) +- bitcoin/bitcoin#15946 Allow maintaining the blockfilterindex when using prune (jonasschnelli) +- bitcoin/bitcoin#18710 Add local thread pool to CCheckQueue (hebasto) +- bitcoin/bitcoin#19521 Coinstats Index (fjahr) +- bitcoin/bitcoin#19806 UTXO snapshot activation (jamesob) +- bitcoin/bitcoin#19905 Remove dead CheckForkWarningConditionsOnNewFork (MarcoFalke) +- bitcoin/bitcoin#19935 Move SaltedHashers to separate file and add some new ones (achow101) +- bitcoin/bitcoin#20054 Remove confusing and useless "unexpected version" warning (MarcoFalke) +- bitcoin/bitcoin#20519 Handle rename failure in `DumpMempool(…)` by using the `RenameOver(…)` return value (practicalswift) +- bitcoin/bitcoin#20749, bitcoin/bitcoin#20750, bitcoin/bitcoin#21055, bitcoin/bitcoin#21270, bitcoin/bitcoin#21525, bitcoin/bitcoin#21391, bitcoin/bitcoin#21767, bitcoin/bitcoin#21866 Prune `g_chainman` usage (dongcarl) +- bitcoin/bitcoin#20833 rpc/validation: enable packages through testmempoolaccept (glozow) +- bitcoin/bitcoin#20834 Locks and docs in ATMP and CheckInputsFromMempoolAndCache (glozow) +- bitcoin/bitcoin#20854 Remove unnecessary try-block (amitiuttarwar) +- bitcoin/bitcoin#20868 Remove redundant check on pindex (jarolrod) +- bitcoin/bitcoin#20921 Don't try to invalidate genesis block in CChainState::InvalidateBlock (theStack) +- bitcoin/bitcoin#20972 Locks: Annotate CTxMemPool::check to require `cs_main` (dongcarl) +- bitcoin/bitcoin#21009 Remove RewindBlockIndex logic (dhruv) +- bitcoin/bitcoin#21025 Guard chainman chainstates with `cs_main` (dongcarl) +- bitcoin/bitcoin#21202 Two small clang lock annotation improvements (amitiuttarwar) +- bitcoin/bitcoin#21523 Run VerifyDB on all chainstates (jamesob) +- bitcoin/bitcoin#21573 Update libsecp256k1 subtree to latest master (sipa) +- bitcoin/bitcoin#21582, bitcoin/bitcoin#21584, bitcoin/bitcoin#21585 Fix assumeutxo crashes (MarcoFalke) +- bitcoin/bitcoin#21681 Fix ActivateSnapshot to use hardcoded nChainTx (jamesob) +- bitcoin/bitcoin#21796 index: Avoid async shutdown on init error (MarcoFalke) +- bitcoin/bitcoin#21946 Document and test lack of inherited signaling in RBF policy (ariard) +- bitcoin/bitcoin#22084 Package testmempoolaccept followups (glozow) +- bitcoin/bitcoin#22102 Remove `Warning:` from warning message printed for unknown new rules (prayank23) +- bitcoin/bitcoin#22112 Force port 0 in I2P (vasild) +- bitcoin/bitcoin#22135 CRegTestParams: Use `args` instead of `gArgs` (kiminuo) +- bitcoin/bitcoin#22146 Reject invalid coin height and output index when loading assumeutxo (MarcoFalke) +- bitcoin/bitcoin#22253 Distinguish between same tx and same-nonwitness-data tx in mempool (glozow) +- bitcoin/bitcoin#22261 Two small fixes to node broadcast logic (jnewbery) +- bitcoin/bitcoin#22415 Make `m_mempool` optional in CChainState (jamesob) +- bitcoin/bitcoin#22499 Update assumed chain params (sriramdvt) +- bitcoin/bitcoin#22589 net, doc: update I2P hardcoded seeds and docs for 22.0 (jonatack) + +### P2P protocol and network code +- bitcoin/bitcoin#18077 Add NAT-PMP port forwarding support (hebasto) +- bitcoin/bitcoin#18722 addrman: improve performance by using more suitable containers (vasild) +- bitcoin/bitcoin#18819 Replace `cs_feeFilter` with simple std::atomic (MarcoFalke) +- bitcoin/bitcoin#19203 Add regression fuzz harness for CVE-2017-18350. Add FuzzedSocket (practicalswift) +- bitcoin/bitcoin#19288 fuzz: Add fuzzing harness for TorController (practicalswift) +- bitcoin/bitcoin#19415 Make DNS lookup mockable, add fuzzing harness (practicalswift) +- bitcoin/bitcoin#19509 Per-Peer Message Capture (troygiorshev) +- bitcoin/bitcoin#19763 Don't try to relay to the address' originator (vasild) +- bitcoin/bitcoin#19771 Replace enum CConnMan::NumConnections with enum class ConnectionDirection (luke-jr) +- bitcoin/bitcoin#19776 net, rpc: expose high bandwidth mode state via getpeerinfo (theStack) +- bitcoin/bitcoin#19832 Put disconnecting logs into BCLog::NET category (hebasto) +- bitcoin/bitcoin#19858 Periodically make block-relay connections and sync headers (sdaftuar) +- bitcoin/bitcoin#19884 No delay in adding fixed seeds if -dnsseed=0 and peers.dat is empty (dhruv) +- bitcoin/bitcoin#20079 Treat handshake misbehavior like unknown message (MarcoFalke) +- bitcoin/bitcoin#20138 Assume that SetCommonVersion is called at most once per peer (MarcoFalke) +- bitcoin/bitcoin#20162 p2p: declare Announcement::m_state as uint8_t, add getter/setter (jonatack) +- bitcoin/bitcoin#20197 Protect onions in AttemptToEvictConnection(), add eviction protection test coverage (jonatack) +- bitcoin/bitcoin#20210 assert `CNode::m_inbound_onion` is inbound in ctor, add getter, unit tests (jonatack) +- bitcoin/bitcoin#20228 addrman: Make addrman a top-level component (jnewbery) +- bitcoin/bitcoin#20234 Don't bind on 0.0.0.0 if binds are restricted to Tor (vasild) +- bitcoin/bitcoin#20477 Add unit testing of node eviction logic (practicalswift) +- bitcoin/bitcoin#20516 Well-defined CAddress disk serialization, and addrv2 anchors.dat (sipa) +- bitcoin/bitcoin#20557 addrman: Fix new table bucketing during unserialization (jnewbery) +- bitcoin/bitcoin#20561 Periodically clear `m_addr_known` (sdaftuar) +- bitcoin/bitcoin#20599 net processing: Tolerate sendheaders and sendcmpct messages before verack (jnewbery) +- bitcoin/bitcoin#20616 Check CJDNS address is valid (lontivero) +- bitcoin/bitcoin#20617 Remove `m_is_manual_connection` from CNodeState (ariard) +- bitcoin/bitcoin#20624 net processing: Remove nStartingHeight check from block relay (jnewbery) +- bitcoin/bitcoin#20651 Make p2p recv buffer timeout 20 minutes for all peers (jnewbery) +- bitcoin/bitcoin#20661 Only select from addrv2-capable peers for torv3 address relay (sipa) +- bitcoin/bitcoin#20685 Add I2P support using I2P SAM (vasild) +- bitcoin/bitcoin#20690 Clean up logging of outbound connection type (sdaftuar) +- bitcoin/bitcoin#20721 Move ping data to `net_processing` (jnewbery) +- bitcoin/bitcoin#20724 Cleanup of -debug=net log messages (ajtowns) +- bitcoin/bitcoin#20747 net processing: Remove dropmessagestest (jnewbery) +- bitcoin/bitcoin#20764 cli -netinfo peer connections dashboard updates 🎄 ✨ (jonatack) +- bitcoin/bitcoin#20788 add RAII socket and use it instead of bare SOCKET (vasild) +- bitcoin/bitcoin#20791 remove unused legacyWhitelisted in AcceptConnection() (jonatack) +- bitcoin/bitcoin#20816 Move RecordBytesSent() call out of `cs_vSend` lock (jnewbery) +- bitcoin/bitcoin#20845 Log to net debug in MaybeDiscourageAndDisconnect except for noban and manual peers (MarcoFalke) +- bitcoin/bitcoin#20864 Move SocketSendData lock annotation to header (MarcoFalke) +- bitcoin/bitcoin#20965 net, rpc: return `NET_UNROUTABLE` as `not_publicly_routable`, automate helps (jonatack) +- bitcoin/bitcoin#20966 banman: save the banlist in a JSON format on disk (vasild) +- bitcoin/bitcoin#21015 Make all of `net_processing` (and some of net) use std::chrono types (dhruv) +- bitcoin/bitcoin#21029 bitcoin-cli: Correct docs (no "generatenewaddress" exists) (luke-jr) +- bitcoin/bitcoin#21148 Split orphan handling from `net_processing` into txorphanage (ajtowns) +- bitcoin/bitcoin#21162 Net Processing: Move RelayTransaction() into PeerManager (jnewbery) +- bitcoin/bitcoin#21167 make `CNode::m_inbound_onion` public, initialize explicitly (jonatack) +- bitcoin/bitcoin#21186 net/net processing: Move addr data into `net_processing` (jnewbery) +- bitcoin/bitcoin#21187 Net processing: Only call PushAddress() from `net_processing` (jnewbery) +- bitcoin/bitcoin#21198 Address outstanding review comments from PR20721 (jnewbery) +- bitcoin/bitcoin#21222 log: Clarify log message when file does not exist (MarcoFalke) +- bitcoin/bitcoin#21235 Clarify disconnect log message in ProcessGetBlockData, remove send bool (MarcoFalke) +- bitcoin/bitcoin#21236 Net processing: Extract `addr` send functionality into MaybeSendAddr() (jnewbery) +- bitcoin/bitcoin#21261 update inbound eviction protection for multiple networks, add I2P peers (jonatack) +- bitcoin/bitcoin#21328 net, refactor: pass uint16 CService::port as uint16 (jonatack) +- bitcoin/bitcoin#21387 Refactor sock to add I2P fuzz and unit tests (vasild) +- bitcoin/bitcoin#21395 Net processing: Remove unused CNodeState.address member (jnewbery) +- bitcoin/bitcoin#21407 i2p: limit the size of incoming messages (vasild) +- bitcoin/bitcoin#21506 p2p, refactor: make NetPermissionFlags an enum class (jonatack) +- bitcoin/bitcoin#21509 Don't send FEEFILTER in blocksonly mode (mzumsande) +- bitcoin/bitcoin#21560 Add Tor v3 hardcoded seeds (laanwj) +- bitcoin/bitcoin#21563 Restrict period when `cs_vNodes` mutex is locked (hebasto) +- bitcoin/bitcoin#21564 Avoid calling getnameinfo when formatting IPv4 addresses in CNetAddr::ToStringIP (practicalswift) +- bitcoin/bitcoin#21631 i2p: always check the return value of Sock::Wait() (vasild) +- bitcoin/bitcoin#21644 p2p, bugfix: use NetPermissions::HasFlag() in CConnman::Bind() (jonatack) +- bitcoin/bitcoin#21659 flag relevant Sock methods with [[nodiscard]] (vasild) +- bitcoin/bitcoin#21750 remove unnecessary check of `CNode::cs_vSend` (vasild) +- bitcoin/bitcoin#21756 Avoid calling `getnameinfo` when formatting IPv6 addresses in `CNetAddr::ToStringIP` (practicalswift) +- bitcoin/bitcoin#21775 Limit `m_block_inv_mutex` (MarcoFalke) +- bitcoin/bitcoin#21825 Add I2P hardcoded seeds (jonatack) +- bitcoin/bitcoin#21843 p2p, rpc: enable GetAddr, GetAddresses, and getnodeaddresses by network (jonatack) +- bitcoin/bitcoin#21845 net processing: Don't require locking `cs_main` before calling RelayTransactions() (jnewbery) +- bitcoin/bitcoin#21872 Sanitize message type for logging (laanwj) +- bitcoin/bitcoin#21914 Use stronger AddLocal() for our I2P address (vasild) +- bitcoin/bitcoin#21985 Return IPv6 scope id in `CNetAddr::ToStringIP()` (laanwj) +- bitcoin/bitcoin#21992 Remove -feefilter option (amadeuszpawlik) +- bitcoin/bitcoin#21996 Pass strings to NetPermissions::TryParse functions by const ref (jonatack) +- bitcoin/bitcoin#22013 ignore block-relay-only peers when skipping DNS seed (ajtowns) +- bitcoin/bitcoin#22050 Remove tor v2 support (jonatack) +- bitcoin/bitcoin#22096 AddrFetch - don't disconnect on self-announcements (mzumsande) +- bitcoin/bitcoin#22141 net processing: Remove hash and fValidatedHeaders from QueuedBlock (jnewbery) +- bitcoin/bitcoin#22144 Randomize message processing peer order (sipa) +- bitcoin/bitcoin#22147 Protect last outbound HB compact block peer (sdaftuar) +- bitcoin/bitcoin#22179 Torv2 removal followups (vasild) +- bitcoin/bitcoin#22211 Relay I2P addresses even if not reachable (by us) (vasild) +- bitcoin/bitcoin#22284 Performance improvements to ProtectEvictionCandidatesByRatio() (jonatack) +- bitcoin/bitcoin#22387 Rate limit the processing of rumoured addresses (sipa) +- bitcoin/bitcoin#22455 addrman: detect on-disk corrupted nNew and nTried during unserialization (vasild) + +### Wallet +- bitcoin/bitcoin#15710 Catch `ios_base::failure` specifically (Bushstar) +- bitcoin/bitcoin#16546 External signer support - Wallet Box edition (Sjors) +- bitcoin/bitcoin#17331 Use effective values throughout coin selection (achow101) +- bitcoin/bitcoin#18418 Increase `OUTPUT_GROUP_MAX_ENTRIES` to 100 (fjahr) +- bitcoin/bitcoin#18842 Mark replaced tx to not be in the mempool anymore (MarcoFalke) +- bitcoin/bitcoin#19136 Add `parent_desc` to `getaddressinfo` (achow101) +- bitcoin/bitcoin#19137 wallettool: Add dump and createfromdump commands (achow101) +- bitcoin/bitcoin#19651 `importdescriptor`s update existing (S3RK) +- bitcoin/bitcoin#20040 Refactor OutputGroups to handle fees and spending eligibility on grouping (achow101) +- bitcoin/bitcoin#20202 Make BDB support optional (achow101) +- bitcoin/bitcoin#20226, bitcoin/bitcoin#21277, - bitcoin/bitcoin#21063 Add `listdescriptors` command (S3RK) +- bitcoin/bitcoin#20267 Disable and fix tests for when BDB is not compiled (achow101) +- bitcoin/bitcoin#20275 List all wallets in non-SQLite and non-BDB builds (ryanofsky) +- bitcoin/bitcoin#20365 wallettool: Add parameter to create descriptors wallet (S3RK) +- bitcoin/bitcoin#20403 `upgradewallet` fixes, improvements, test coverage (jonatack) +- bitcoin/bitcoin#20448 `unloadwallet`: Allow specifying `wallet_name` param matching RPC endpoint wallet (luke-jr) +- bitcoin/bitcoin#20536 Error with "Transaction too large" if the funded tx will end up being too large after signing (achow101) +- bitcoin/bitcoin#20687 Add missing check for -descriptors wallet tool option (MarcoFalke) +- bitcoin/bitcoin#20952 Add BerkeleyDB version sanity check at init time (laanwj) +- bitcoin/bitcoin#21127 Load flags before everything else (Sjors) +- bitcoin/bitcoin#21141 Add new format string placeholders for walletnotify (maayank) +- bitcoin/bitcoin#21238 A few descriptor improvements to prepare for Taproot support (sipa) +- bitcoin/bitcoin#21302 `createwallet` examples for descriptor wallets (S3RK) +- bitcoin/bitcoin#21329 descriptor wallet: Cache last hardened xpub and use in normalized descriptors (achow101) +- bitcoin/bitcoin#21365 Basic Taproot signing support for descriptor wallets (sipa) +- bitcoin/bitcoin#21417 Misc external signer improvement and HWI 2 support (Sjors) +- bitcoin/bitcoin#21467 Move external signer out of wallet module (Sjors) +- bitcoin/bitcoin#21572 Fix wrong wallet RPC context set after #21366 (ryanofsky) +- bitcoin/bitcoin#21574 Drop JSONRPCRequest constructors after #21366 (ryanofsky) +- bitcoin/bitcoin#21666 Miscellaneous external signer changes (fanquake) +- bitcoin/bitcoin#21759 Document coin selection code (glozow) +- bitcoin/bitcoin#21786 Ensure sat/vB feerates are in range (mantissa of 3) (jonatack) +- bitcoin/bitcoin#21944 Fix issues when `walletdir` is root directory (prayank23) +- bitcoin/bitcoin#22042 Replace size/weight estimate tuple with struct for named fields (instagibbs) +- bitcoin/bitcoin#22051 Basic Taproot derivation support for descriptors (sipa) +- bitcoin/bitcoin#22154 Add OutputType::BECH32M and related wallet support for fetching bech32m addresses (achow101) +- bitcoin/bitcoin#22156 Allow tr() import only when Taproot is active (achow101) +- bitcoin/bitcoin#22166 Add support for inferring tr() descriptors (sipa) +- bitcoin/bitcoin#22173 Do not load external signers wallets when unsupported (achow101) +- bitcoin/bitcoin#22308 Add missing BlockUntilSyncedToCurrentChain (MarcoFalke) +- bitcoin/bitcoin#22334 Do not spam about non-existent spk managers (S3RK) +- bitcoin/bitcoin#22379 Erase spkmans rather than setting to nullptr (achow101) +- bitcoin/bitcoin#22421 Make IsSegWitOutput return true for taproot outputs (sipa) +- bitcoin/bitcoin#22461 Change ScriptPubKeyMan::Upgrade default to True (achow101) +- bitcoin/bitcoin#22492 Reorder locks in dumpwallet to avoid lock order assertion (achow101) +- bitcoin/bitcoin#22686 Use GetSelectionAmount in ApproximateBestSubset (achow101) + +### RPC and other APIs +- bitcoin/bitcoin#18335, bitcoin/bitcoin#21484 cli: Print useful error if bitcoind rpc work queue exceeded (LarryRuane) +- bitcoin/bitcoin#18466 Fix invalid parameter error codes for `{sign,verify}message` RPCs (theStack) +- bitcoin/bitcoin#18772 Calculate fees in `getblock` using BlockUndo data (robot-visions) +- bitcoin/bitcoin#19033 http: Release work queue after event base finish (promag) +- bitcoin/bitcoin#19055 Add MuHash3072 implementation (fjahr) +- bitcoin/bitcoin#19145 Add `hash_type` MUHASH for gettxoutsetinfo (fjahr) +- bitcoin/bitcoin#19847 Avoid duplicate set lookup in `gettxoutproof` (promag) +- bitcoin/bitcoin#20286 Deprecate `addresses` and `reqSigs` from RPC outputs (mjdietzx) +- bitcoin/bitcoin#20459 Fail to return undocumented return values (MarcoFalke) +- bitcoin/bitcoin#20461 Validate `-rpcauth` arguments (promag) +- bitcoin/bitcoin#20556 Properly document return values (`submitblock`, `gettxout`, `getblocktemplate`, `scantxoutset`) (MarcoFalke) +- bitcoin/bitcoin#20755 Remove deprecated fields from `getpeerinfo` (amitiuttarwar) +- bitcoin/bitcoin#20832 Better error messages for invalid addresses (eilx2) +- bitcoin/bitcoin#20867 Support up to 20 keys for multisig under Segwit context (darosior) +- bitcoin/bitcoin#20877 cli: `-netinfo` user help and argument parsing improvements (jonatack) +- bitcoin/bitcoin#20891 Remove deprecated bumpfee behavior (achow101) +- bitcoin/bitcoin#20916 Return wtxid from `testmempoolaccept` (MarcoFalke) +- bitcoin/bitcoin#20917 Add missing signet mentions in network name lists (theStack) +- bitcoin/bitcoin#20941 Document `RPC_TRANSACTION_ALREADY_IN_CHAIN` exception (jarolrod) +- bitcoin/bitcoin#20944 Return total fee in `getmempoolinfo` (MarcoFalke) +- bitcoin/bitcoin#20964 Add specific error code for "wallet already loaded" (laanwj) +- bitcoin/bitcoin#21053 Document {previous,next}blockhash as optional (theStack) +- bitcoin/bitcoin#21056 Add a `-rpcwaittimeout` parameter to limit time spent waiting (cdecker) +- bitcoin/bitcoin#21192 cli: Treat high detail levels as maximum in `-netinfo` (laanwj) +- bitcoin/bitcoin#21311 Document optional fields for `getchaintxstats` result (theStack) +- bitcoin/bitcoin#21359 `include_unsafe` option for fundrawtransaction (t-bast) +- bitcoin/bitcoin#21426 Remove `scantxoutset` EXPERIMENTAL warning (jonatack) +- bitcoin/bitcoin#21544 Missing doc updates for bumpfee psbt update (MarcoFalke) +- bitcoin/bitcoin#21594 Add `network` field to `getnodeaddresses` (jonatack) +- bitcoin/bitcoin#21595, bitcoin/bitcoin#21753 cli: Create `-addrinfo` (jonatack) +- bitcoin/bitcoin#21602 Add additional ban time fields to `listbanned` (jarolrod) +- bitcoin/bitcoin#21679 Keep default argument value in correct type (promag) +- bitcoin/bitcoin#21718 Improve error message for `getblock` invalid datatype (klementtan) +- bitcoin/bitcoin#21913 RPCHelpMan fixes (kallewoof) +- bitcoin/bitcoin#22021 `bumpfee`/`psbtbumpfee` fixes and updates (jonatack) +- bitcoin/bitcoin#22043 `addpeeraddress` test coverage, code simplify/constness (jonatack) +- bitcoin/bitcoin#22327 cli: Avoid truncating `-rpcwaittimeout` (MarcoFalke) + +### GUI +- bitcoin/bitcoin#18948 Call setParent() in the parent's context (hebasto) +- bitcoin/bitcoin#20482 Add depends qt fix for ARM macs (jonasschnelli) +- bitcoin/bitcoin#21836 scripted-diff: Replace three dots with ellipsis in the ui strings (hebasto) +- bitcoin/bitcoin#21935 Enable external signer support for GUI builds (Sjors) +- bitcoin/bitcoin#22133 Make QWindowsVistaStylePlugin available again (regression) (hebasto) +- bitcoin-core/gui#4 UI external signer support (e.g. hardware wallet) (Sjors) +- bitcoin-core/gui#13 Hide peer detail view if multiple are selected (promag) +- bitcoin-core/gui#18 Add peertablesortproxy module (hebasto) +- bitcoin-core/gui#21 Improve pruning tooltip (fluffypony, BitcoinErrorLog) +- bitcoin-core/gui#72 Log static plugins meta data and used style (hebasto) +- bitcoin-core/gui#79 Embed monospaced font (hebasto) +- bitcoin-core/gui#85 Remove unused "What's This" button in dialogs on Windows OS (hebasto) +- bitcoin-core/gui#115 Replace "Hide tray icon" option with positive "Show tray icon" one (hebasto) +- bitcoin-core/gui#118 Remove BDB version from the Information tab (hebasto) +- bitcoin-core/gui#121 Early subscribe core signals in transaction table model (promag) +- bitcoin-core/gui#123 Do not accept command while executing another one (hebasto) +- bitcoin-core/gui#125 Enable changing the autoprune block space size in intro dialog (luke-jr) +- bitcoin-core/gui#138 Unlock encrypted wallet "OK" button bugfix (mjdietzx) +- bitcoin-core/gui#139 doc: Improve gui/src/qt README.md (jarolrod) +- bitcoin-core/gui#154 Support macOS Dark mode (goums, Uplab) +- bitcoin-core/gui#162 Add network to peers window and peer details (jonatack) +- bitcoin-core/gui#163, bitcoin-core/gui#180 Peer details: replace Direction with Connection Type (jonatack) +- bitcoin-core/gui#164 Handle peer addition/removal in a right way (hebasto) +- bitcoin-core/gui#165 Save QSplitter state in QSettings (hebasto) +- bitcoin-core/gui#173 Follow Qt docs when implementing rowCount and columnCount (hebasto) +- bitcoin-core/gui#179 Add Type column to peers window, update peer details name/tooltip (jonatack) +- bitcoin-core/gui#186 Add information to "Confirm fee bump" window (prayank23) +- bitcoin-core/gui#189 Drop workaround for QTBUG-42503 which was fixed in Qt 5.5.0 (prusnak) +- bitcoin-core/gui#194 Save/restore RPCConsole geometry only for window (hebasto) +- bitcoin-core/gui#202 Fix right panel toggle in peers tab (RandyMcMillan) +- bitcoin-core/gui#203 Display plain "Inbound" in peer details (jonatack) +- bitcoin-core/gui#204 Drop buggy TableViewLastColumnResizingFixer class (hebasto) +- bitcoin-core/gui#205, bitcoin-core/gui#229 Save/restore TransactionView and recentRequestsView tables column sizes (hebasto) +- bitcoin-core/gui#206 Display fRelayTxes and `bip152_highbandwidth_{to, from}` in peer details (jonatack) +- bitcoin-core/gui#213 Add Copy Address Action to Payment Requests (jarolrod) +- bitcoin-core/gui#214 Disable requests context menu actions when appropriate (jarolrod) +- bitcoin-core/gui#217 Make warning label look clickable (jarolrod) +- bitcoin-core/gui#219 Prevent the main window popup menu (hebasto) +- bitcoin-core/gui#220 Do not translate file extensions (hebasto) +- bitcoin-core/gui#221 RPCConsole translatable string fixes and improvements (jonatack) +- bitcoin-core/gui#226 Add "Last Block" and "Last Tx" rows to peer details area (jonatack) +- bitcoin-core/gui#233 qt test: Don't bind to regtest port (achow101) +- bitcoin-core/gui#243 Fix issue when disabling the auto-enabled blank wallet checkbox (jarolrod) +- bitcoin-core/gui#246 Revert "qt: Use "fusion" style on macOS Big Sur with old Qt" (hebasto) +- bitcoin-core/gui#248 For values of "Bytes transferred" and "Bytes/s" with 1000-based prefix names use 1000-based divisor instead of 1024-based (wodry) +- bitcoin-core/gui#251 Improve URI/file handling message (hebasto) +- bitcoin-core/gui#256 Save/restore column sizes of the tables in the Peers tab (hebasto) +- bitcoin-core/gui#260 Handle exceptions isntead of crash (hebasto) +- bitcoin-core/gui#263 Revamp context menus (hebasto) +- bitcoin-core/gui#271 Don't clear console prompt when font resizing (jarolrod) +- bitcoin-core/gui#275 Support runtime appearance adjustment on macOS (hebasto) +- bitcoin-core/gui#276 Elide long strings in their middle in the Peers tab (hebasto) +- bitcoin-core/gui#281 Set shortcuts for console's resize buttons (jarolrod) +- bitcoin-core/gui#293 Enable wordWrap for Services (RandyMcMillan) +- bitcoin-core/gui#296 Do not use QObject::tr plural syntax for numbers with a unit symbol (hebasto) +- bitcoin-core/gui#297 Avoid unnecessary translations (hebasto) +- bitcoin-core/gui#298 Peertableview alternating row colors (RandyMcMillan) +- bitcoin-core/gui#300 Remove progress bar on modal overlay (brunoerg) +- bitcoin-core/gui#309 Add access to the Peers tab from the network icon (hebasto) +- bitcoin-core/gui#311 Peers Window rename 'Peer id' to 'Peer' (jarolrod) +- bitcoin-core/gui#313 Optimize string concatenation by default (hebasto) +- bitcoin-core/gui#325 Align numbers in the "Peer Id" column to the right (hebasto) +- bitcoin-core/gui#329 Make console buttons look clickable (jarolrod) +- bitcoin-core/gui#330 Allow prompt icon to be colorized (jarolrod) +- bitcoin-core/gui#331 Make RPC console welcome message translation-friendly (hebasto) +- bitcoin-core/gui#332 Replace disambiguation strings with translator comments (hebasto) +- bitcoin-core/gui#335 test: Use QSignalSpy instead of QEventLoop (jarolrod) +- bitcoin-core/gui#343 Improve the GUI responsiveness when progress dialogs are used (hebasto) +- bitcoin-core/gui#361 Fix GUI segfault caused by bitcoin/bitcoin#22216 (ryanofsky) +- bitcoin-core/gui#362 Add keyboard shortcuts to context menus (luke-jr) +- bitcoin-core/gui#366 Dark Mode fixes/portability (luke-jr) +- bitcoin-core/gui#375 Emit dataChanged signal to dynamically re-sort Peers table (hebasto) +- bitcoin-core/gui#393 Fix regression in "Encrypt Wallet" menu item (hebasto) +- bitcoin-core/gui#396 Ensure external signer option remains disabled without signers (achow101) +- bitcoin-core/gui#406 Handle new added plurals in `bitcoin_en.ts` (hebasto) + +### Build system +- bitcoin/bitcoin#17227 Add Android packaging support (icota) +- bitcoin/bitcoin#17920 guix: Build support for macOS (dongcarl) +- bitcoin/bitcoin#18298 Fix Qt processing of configure script for depends with DEBUG=1 (hebasto) +- bitcoin/bitcoin#19160 multiprocess: Add basic spawn and IPC support (ryanofsky) +- bitcoin/bitcoin#19504 Bump minimum python version to 3.6 (ajtowns) +- bitcoin/bitcoin#19522 fix building libconsensus with reduced exports for Darwin targets (fanquake) +- bitcoin/bitcoin#19683 Pin clang search paths for darwin host (dongcarl) +- bitcoin/bitcoin#19764 Split boost into build/host packages + bump + cleanup (dongcarl) +- bitcoin/bitcoin#19817 libtapi 1100.0.11 (fanquake) +- bitcoin/bitcoin#19846 enable unused member function diagnostic (Zero-1729) +- bitcoin/bitcoin#19867 Document and cleanup Qt hacks (fanquake) +- bitcoin/bitcoin#20046 Set `CMAKE_INSTALL_RPATH` for native packages (ryanofsky) +- bitcoin/bitcoin#20223 Drop the leading 0 from the version number (achow101) +- bitcoin/bitcoin#20333 Remove `native_biplist` dependency (fanquake) +- bitcoin/bitcoin#20353 configure: Support -fdebug-prefix-map and -fmacro-prefix-map (ajtowns) +- bitcoin/bitcoin#20359 Various config.site.in improvements and linting (dongcarl) +- bitcoin/bitcoin#20413 Require C++17 compiler (MarcoFalke) +- bitcoin/bitcoin#20419 Set minimum supported macOS to 10.14 (fanquake) +- bitcoin/bitcoin#20421 miniupnpc 2.2.2 (fanquake) +- bitcoin/bitcoin#20422 Mac deployment unification (fanquake) +- bitcoin/bitcoin#20424 Update univalue subtree (MarcoFalke) +- bitcoin/bitcoin#20449 Fix Windows installer build (achow101) +- bitcoin/bitcoin#20468 Warn when generating man pages for binaries built from a dirty branch (tylerchambers) +- bitcoin/bitcoin#20469 Avoid secp256k1.h include from system (dergoegge) +- bitcoin/bitcoin#20470 Replace genisoimage with xorriso (dongcarl) +- bitcoin/bitcoin#20471 Use C++17 in depends (fanquake) +- bitcoin/bitcoin#20496 Drop unneeded macOS framework dependencies (hebasto) +- bitcoin/bitcoin#20520 Do not force Precompiled Headers (PCH) for building Qt on Linux (hebasto) +- bitcoin/bitcoin#20549 Support make src/bitcoin-node and src/bitcoin-gui (promag) +- bitcoin/bitcoin#20565 Ensure PIC build for bdb on Android (BlockMechanic) +- bitcoin/bitcoin#20594 Fix getauxval calls in randomenv.cpp (jonasschnelli) +- bitcoin/bitcoin#20603 Update crc32c subtree (MarcoFalke) +- bitcoin/bitcoin#20609 configure: output notice that test binary is disabled by fuzzing (apoelstra) +- bitcoin/bitcoin#20619 guix: Quality of life improvements (dongcarl) +- bitcoin/bitcoin#20629 Improve id string robustness (dongcarl) +- bitcoin/bitcoin#20641 Use Qt top-level build facilities (hebasto) +- bitcoin/bitcoin#20650 Drop workaround for a fixed bug in Qt build system (hebasto) +- bitcoin/bitcoin#20673 Use more legible qmake commands in qt package (hebasto) +- bitcoin/bitcoin#20684 Define .INTERMEDIATE target once only (hebasto) +- bitcoin/bitcoin#20720 more robustly check for fcf-protection support (fanquake) +- bitcoin/bitcoin#20734 Make platform-specific targets available for proper platform builds only (hebasto) +- bitcoin/bitcoin#20936 build fuzz tests by default (danben) +- bitcoin/bitcoin#20937 guix: Make nsis reproducible by respecting SOURCE-DATE-EPOCH (dongcarl) +- bitcoin/bitcoin#20938 fix linking against -latomic when building for riscv (fanquake) +- bitcoin/bitcoin#20939 fix `RELOC_SECTION` security check for bitcoin-util (fanquake) +- bitcoin/bitcoin#20963 gitian-linux: Build binaries for 64-bit POWER (continued) (laanwj) +- bitcoin/bitcoin#21036 gitian: Bump descriptors to focal for 22.0 (fanquake) +- bitcoin/bitcoin#21045 Adds switch to enable/disable randomized base address in MSVC builds (EthanHeilman) +- bitcoin/bitcoin#21065 make macOS HOST in download-osx generic (fanquake) +- bitcoin/bitcoin#21078 guix: only download sources for hosts being built (fanquake) +- bitcoin/bitcoin#21116 Disable --disable-fuzz-binary for gitian/guix builds (hebasto) +- bitcoin/bitcoin#21182 remove mostly pointless `BOOST_PROCESS` macro (fanquake) +- bitcoin/bitcoin#21205 actually fail when Boost is missing (fanquake) +- bitcoin/bitcoin#21209 use newer source for libnatpmp (fanquake) +- bitcoin/bitcoin#21226 Fix fuzz binary compilation under windows (danben) +- bitcoin/bitcoin#21231 Add /opt/homebrew to path to look for boost libraries (fyquah) +- bitcoin/bitcoin#21239 guix: Add codesignature attachment support for osx+win (dongcarl) +- bitcoin/bitcoin#21250 Make `HAVE_O_CLOEXEC` available outside LevelDB (bugfix) (theStack) +- bitcoin/bitcoin#21272 guix: Passthrough `SDK_PATH` into container (dongcarl) +- bitcoin/bitcoin#21274 assumptions: Assume C++17 (fanquake) +- bitcoin/bitcoin#21286 Bump minimum Qt version to 5.9.5 (hebasto) +- bitcoin/bitcoin#21298 guix: Bump time-machine, glibc, and linux-headers (dongcarl) +- bitcoin/bitcoin#21304 guix: Add guix-clean script + establish gc-root for container profiles (dongcarl) +- bitcoin/bitcoin#21320 fix libnatpmp macos cross compile (fanquake) +- bitcoin/bitcoin#21321 guix: Add curl to required tool list (hebasto) +- bitcoin/bitcoin#21333 set Unicode true for NSIS installer (fanquake) +- bitcoin/bitcoin#21339 Make `AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER])` unconditional (hebasto) +- bitcoin/bitcoin#21349 Fix fuzz-cuckoocache cross-compiling with DEBUG=1 (hebasto) +- bitcoin/bitcoin#21354 build, doc: Drop no longer required packages from macOS cross-compiling dependencies (hebasto) +- bitcoin/bitcoin#21363 build, qt: Improve Qt static plugins/libs check code (hebasto) +- bitcoin/bitcoin#21375 guix: Misc feedback-based fixes + hier restructuring (dongcarl) +- bitcoin/bitcoin#21376 Qt 5.12.10 (fanquake) +- bitcoin/bitcoin#21382 Clean remnants of QTBUG-34748 fix (hebasto) +- bitcoin/bitcoin#21400 Fix regression introduced in #21363 (hebasto) +- bitcoin/bitcoin#21403 set --build when configuring packages in depends (fanquake) +- bitcoin/bitcoin#21421 don't try and use -fstack-clash-protection on Windows (fanquake) +- bitcoin/bitcoin#21423 Cleanups and follow ups after bumping Qt to 5.12.10 (hebasto) +- bitcoin/bitcoin#21427 Fix `id_string` invocations (dongcarl) +- bitcoin/bitcoin#21430 Add -Werror=implicit-fallthrough compile flag (hebasto) +- bitcoin/bitcoin#21457 Split libtapi and clang out of `native_cctools` (fanquake) +- bitcoin/bitcoin#21462 guix: Add guix-{attest,verify} scripts (dongcarl) +- bitcoin/bitcoin#21495 build, qt: Fix static builds on macOS Big Sur (hebasto) +- bitcoin/bitcoin#21497 Do not opt-in unused CoreWLAN stuff in depends for macOS (hebasto) +- bitcoin/bitcoin#21543 Enable safe warnings for msvc builds (hebasto) +- bitcoin/bitcoin#21565 Make `bitcoin_qt.m4` more generic (fanquake) +- bitcoin/bitcoin#21610 remove -Wdeprecated-register from NOWARN flags (fanquake) +- bitcoin/bitcoin#21613 enable -Wdocumentation (fanquake) +- bitcoin/bitcoin#21629 Fix configuring when building depends with `NO_BDB=1` (fanquake) +- bitcoin/bitcoin#21654 build, qt: Make Qt rcc output always deterministic (hebasto) +- bitcoin/bitcoin#21655 build, qt: No longer need to set `QT_RCC_TEST=1` for determinism (hebasto) +- bitcoin/bitcoin#21658 fix make deploy for arm64-darwin (sgulls) +- bitcoin/bitcoin#21694 Use XLIFF file to provide more context to Transifex translators (hebasto) +- bitcoin/bitcoin#21708, bitcoin/bitcoin#21593 Drop pointless sed commands (hebasto) +- bitcoin/bitcoin#21731 Update msvc build to use Qt5.12.10 binaries (sipsorcery) +- bitcoin/bitcoin#21733 Re-add command to install vcpkg (dplusplus1024) +- bitcoin/bitcoin#21793 Use `-isysroot` over `--sysroot` on macOS (fanquake) +- bitcoin/bitcoin#21869 Add missing `-D_LIBCPP_DEBUG=1` to debug flags (MarcoFalke) +- bitcoin/bitcoin#21889 macho: check for control flow instrumentation (fanquake) +- bitcoin/bitcoin#21920 Improve macro for testing -latomic requirement (MarcoFalke) +- bitcoin/bitcoin#21991 libevent 2.1.12-stable (fanquake) +- bitcoin/bitcoin#22054 Bump Qt version to 5.12.11 (hebasto) +- bitcoin/bitcoin#22063 Use Qt archive of the same version as the compiled binaries (hebasto) +- bitcoin/bitcoin#22070 Don't use cf-protection when targeting arm-apple-darwin (fanquake) +- bitcoin/bitcoin#22071 Latest config.guess and config.sub (fanquake) +- bitcoin/bitcoin#22075 guix: Misc leftover usability improvements (dongcarl) +- bitcoin/bitcoin#22123 Fix qt.mk for mac arm64 (promag) +- bitcoin/bitcoin#22174 build, qt: Fix libraries linking order for Linux hosts (hebasto) +- bitcoin/bitcoin#22182 guix: Overhaul how guix-{attest,verify} works and hierarchy (dongcarl) +- bitcoin/bitcoin#22186 build, qt: Fix compiling qt package in depends with GCC 11 (hebasto) +- bitcoin/bitcoin#22199 macdeploy: minor fixups and simplifications (fanquake) +- bitcoin/bitcoin#22230 Fix MSVC linker /SubSystem option for bitcoin-qt.exe (hebasto) +- bitcoin/bitcoin#22234 Mark print-% target as phony (dgoncharov) +- bitcoin/bitcoin#22238 improve detection of eBPF support (fanquake) +- bitcoin/bitcoin#22258 Disable deprecated-copy warning only when external warnings are enabled (MarcoFalke) +- bitcoin/bitcoin#22320 set minimum required Boost to 1.64.0 (fanquake) +- bitcoin/bitcoin#22348 Fix cross build for Windows with Boost Process (hebasto) +- bitcoin/bitcoin#22365 guix: Avoid relying on newer symbols by rebasing our cross toolchains on older glibcs (dongcarl) +- bitcoin/bitcoin#22381 guix: Test security-check sanity before performing them (with macOS) (fanquake) +- bitcoin/bitcoin#22405 Remove --enable-glibc-back-compat from Guix build (fanquake) +- bitcoin/bitcoin#22406 Remove --enable-determinism configure option (fanquake) +- bitcoin/bitcoin#22410 Avoid GCC 7.1 ABI change warning in guix build (sipa) +- bitcoin/bitcoin#22436 use aarch64 Clang if cross-compiling for darwin on aarch64 (fanquake) +- bitcoin/bitcoin#22465 guix: Pin kernel-header version, time-machine to upstream 1.3.0 commit (dongcarl) +- bitcoin/bitcoin#22511 guix: Silence `getent(1)` invocation, doc fixups (dongcarl) +- bitcoin/bitcoin#22531 guix: Fixes to guix-{attest,verify} (achow101) +- bitcoin/bitcoin#22642 release: Release with separate sha256sums and sig files (dongcarl) +- bitcoin/bitcoin#22685 clientversion: No suffix `#if CLIENT_VERSION_IS_RELEASE` (dongcarl) +- bitcoin/bitcoin#22713 Fix build with Boost 1.77.0 (sizeofvoid) + +### Tests and QA +- bitcoin/bitcoin#14604 Add test and refactor `feature_block.py` (sanket1729) +- bitcoin/bitcoin#17556 Change `feature_config_args.py` not to rely on strange regtest=0 behavior (ryanofsky) +- bitcoin/bitcoin#18795 wallet issue with orphaned rewards (domob1812) +- bitcoin/bitcoin#18847 compressor: Use a prevector in CompressScript serialization (jb55) +- bitcoin/bitcoin#19259 fuzz: Add fuzzing harness for LoadMempool(…) and DumpMempool(…) (practicalswift) +- bitcoin/bitcoin#19315 Allow outbound & block-relay-only connections in functional tests. (amitiuttarwar) +- bitcoin/bitcoin#19698 Apply strict verification flags for transaction tests and assert backwards compatibility (glozow) +- bitcoin/bitcoin#19801 Check for all possible `OP_CLTV` fail reasons in `feature_cltv.py` (BIP 65) (theStack) +- bitcoin/bitcoin#19893 Remove or explain syncwithvalidationinterfacequeue (MarcoFalke) +- bitcoin/bitcoin#19972 fuzz: Add fuzzing harness for node eviction logic (practicalswift) +- bitcoin/bitcoin#19982 Fix inconsistent lock order in `wallet_tests/CreateWallet` (hebasto) +- bitcoin/bitcoin#20000 Fix creation of "std::string"s with \0s (vasild) +- bitcoin/bitcoin#20047 Use `wait_for_{block,header}` helpers in `p2p_fingerprint.py` (theStack) +- bitcoin/bitcoin#20171 Add functional test `test_txid_inv_delay` (ariard) +- bitcoin/bitcoin#20189 Switch to BIP341's suggested scheme for outputs without script (sipa) +- bitcoin/bitcoin#20248 Fix length of R check in `key_signature_tests` (dgpv) +- bitcoin/bitcoin#20276, bitcoin/bitcoin#20385, bitcoin/bitcoin#20688, bitcoin/bitcoin#20692 Run various mempool tests even with wallet disabled (mjdietzx) +- bitcoin/bitcoin#20323 Create or use existing properly initialized NodeContexts (dongcarl) +- bitcoin/bitcoin#20354 Add `feature_taproot.py --previous_release` (MarcoFalke) +- bitcoin/bitcoin#20370 fuzz: Version handshake (MarcoFalke) +- bitcoin/bitcoin#20377 fuzz: Fill various small fuzzing gaps (practicalswift) +- bitcoin/bitcoin#20425 fuzz: Make CAddrMan fuzzing harness deterministic (practicalswift) +- bitcoin/bitcoin#20430 Sanitizers: Add suppression for unsigned-integer-overflow in libstdc++ (jonasschnelli) +- bitcoin/bitcoin#20437 fuzz: Avoid time-based "non-determinism" in fuzzing harnesses by using mocked GetTime() (practicalswift) +- bitcoin/bitcoin#20458 Add `is_bdb_compiled` helper (Sjors) +- bitcoin/bitcoin#20466 Fix intermittent `p2p_fingerprint` issue (MarcoFalke) +- bitcoin/bitcoin#20472 Add testing of ParseInt/ParseUInt edge cases with leading +/-/0:s (practicalswift) +- bitcoin/bitcoin#20507 sync: print proper lock order location when double lock is detected (vasild) +- bitcoin/bitcoin#20522 Fix sync issue in `disconnect_p2ps` (amitiuttarwar) +- bitcoin/bitcoin#20524 Move `MIN_VERSION_SUPPORTED` to p2p.py (jnewbery) +- bitcoin/bitcoin#20540 Fix `wallet_multiwallet` issue on windows (MarcoFalke) +- bitcoin/bitcoin#20560 fuzz: Link all targets once (MarcoFalke) +- bitcoin/bitcoin#20567 Add option to git-subtree-check to do full check, add help (laanwj) +- bitcoin/bitcoin#20569 Fix intermittent `wallet_multiwallet` issue with `got_loading_error` (MarcoFalke) +- bitcoin/bitcoin#20613 Use Popen.wait instead of RPC in `assert_start_raises_init_error` (MarcoFalke) +- bitcoin/bitcoin#20663 fuzz: Hide `script_assets_test_minimizer` (MarcoFalke) +- bitcoin/bitcoin#20674 fuzz: Call SendMessages after ProcessMessage to increase coverage (MarcoFalke) +- bitcoin/bitcoin#20683 Fix restart node race (MarcoFalke) +- bitcoin/bitcoin#20686 fuzz: replace CNode code with fuzz/util.h::ConsumeNode() (jonatack) +- bitcoin/bitcoin#20733 Inline non-member functions with body in fuzzing headers (pstratem) +- bitcoin/bitcoin#20737 Add missing assignment in `mempool_resurrect.py` (MarcoFalke) +- bitcoin/bitcoin#20745 Correct `epoll_ctl` data race suppression (hebasto) +- bitcoin/bitcoin#20748 Add race:SendZmqMessage tsan suppression (MarcoFalke) +- bitcoin/bitcoin#20760 Set correct nValue for multi-op-return policy check (MarcoFalke) +- bitcoin/bitcoin#20761 fuzz: Check that `NULL_DATA` is unspendable (MarcoFalke) +- bitcoin/bitcoin#20765 fuzz: Check that certain script TxoutType are nonstandard (mjdietzx) +- bitcoin/bitcoin#20772 fuzz: Bolster ExtractDestination(s) checks (mjdietzx) +- bitcoin/bitcoin#20789 fuzz: Rework strong and weak net enum fuzzing (MarcoFalke) +- bitcoin/bitcoin#20828 fuzz: Introduce CallOneOf helper to replace switch-case (MarcoFalke) +- bitcoin/bitcoin#20839 fuzz: Avoid extraneous copy of input data, using Span<> (MarcoFalke) +- bitcoin/bitcoin#20844 Add sanitizer suppressions for AMD EPYC CPUs (MarcoFalke) +- bitcoin/bitcoin#20857 Update documentation in `feature_csv_activation.py` (PiRK) +- bitcoin/bitcoin#20876 Replace getmempoolentry with testmempoolaccept in MiniWallet (MarcoFalke) +- bitcoin/bitcoin#20881 fuzz: net permission flags in net processing (MarcoFalke) +- bitcoin/bitcoin#20882 fuzz: Add missing muhash registration (MarcoFalke) +- bitcoin/bitcoin#20908 fuzz: Use mocktime in `process_message*` fuzz targets (MarcoFalke) +- bitcoin/bitcoin#20915 fuzz: Fail if message type is not fuzzed (MarcoFalke) +- bitcoin/bitcoin#20946 fuzz: Consolidate fuzzing TestingSetup initialization (dongcarl) +- bitcoin/bitcoin#20954 Declare `nodes` type `in test_framework.py` (kiminuo) +- bitcoin/bitcoin#20955 Fix `get_previous_releases.py` for aarch64 (MarcoFalke) +- bitcoin/bitcoin#20969 check that getblockfilter RPC fails without block filter index (theStack) +- bitcoin/bitcoin#20971 Work around libFuzzer deadlock (MarcoFalke) +- bitcoin/bitcoin#20993 Store subversion (user agent) as string in `msg_version` (theStack) +- bitcoin/bitcoin#20995 fuzz: Avoid initializing version to less than `MIN_PEER_PROTO_VERSION` (MarcoFalke) +- bitcoin/bitcoin#20998 Fix BlockToJsonVerbose benchmark (martinus) +- bitcoin/bitcoin#21003 Move MakeNoLogFileContext to `libtest_util`, and use it in bench (MarcoFalke) +- bitcoin/bitcoin#21008 Fix zmq test flakiness, improve speed (theStack) +- bitcoin/bitcoin#21023 fuzz: Disable shuffle when merge=1 (MarcoFalke) +- bitcoin/bitcoin#21037 fuzz: Avoid designated initialization (C++20) in fuzz tests (practicalswift) +- bitcoin/bitcoin#21042 doc, test: Improve `setup_clean_chain` documentation (fjahr) +- bitcoin/bitcoin#21080 fuzz: Configure check for main function (take 2) (MarcoFalke) +- bitcoin/bitcoin#21084 Fix timeout decrease in `feature_assumevalid` (brunoerg) +- bitcoin/bitcoin#21096 Re-add dead code detection (flack) +- bitcoin/bitcoin#21100 Remove unused function `xor_bytes` (theStack) +- bitcoin/bitcoin#21115 Fix Windows cross build (hebasto) +- bitcoin/bitcoin#21117 Remove `assert_blockchain_height` (MarcoFalke) +- bitcoin/bitcoin#21121 Small unit test improvements, including helper to make mempool transaction (amitiuttarwar) +- bitcoin/bitcoin#21124 Remove unnecessary assignment in bdb (brunoerg) +- bitcoin/bitcoin#21125 Change `BOOST_CHECK` to `BOOST_CHECK_EQUAL` for paths (kiminuo) +- bitcoin/bitcoin#21142, bitcoin/bitcoin#21512 fuzz: Add `tx_pool` fuzz target (MarcoFalke) +- bitcoin/bitcoin#21165 Use mocktime in `test_seed_peers` (dhruv) +- bitcoin/bitcoin#21169 fuzz: Add RPC interface fuzzing. Increase fuzzing coverage from 65% to 70% (practicalswift) +- bitcoin/bitcoin#21170 bench: Add benchmark to write json into a string (martinus) +- bitcoin/bitcoin#21178 Run `mempool_reorg.py` even with wallet disabled (DariusParvin) +- bitcoin/bitcoin#21185 fuzz: Remove expensive and redundant muhash from crypto fuzz target (MarcoFalke) +- bitcoin/bitcoin#21200 Speed up `rpc_blockchain.py` by removing miniwallet.generate() (MarcoFalke) +- bitcoin/bitcoin#21211 Move `P2WSH_OP_TRUE` to shared test library (MarcoFalke) +- bitcoin/bitcoin#21228 Avoid comparision of integers with different signs (jonasschnelli) +- bitcoin/bitcoin#21230 Fix `NODE_NETWORK_LIMITED_MIN_BLOCKS` disconnection (MarcoFalke) +- bitcoin/bitcoin#21252 Add missing wait for sync to `feature_blockfilterindex_prune` (MarcoFalke) +- bitcoin/bitcoin#21254 Avoid connecting to real network when running tests (MarcoFalke) +- bitcoin/bitcoin#21264 fuzz: Two scripted diff renames (MarcoFalke) +- bitcoin/bitcoin#21280 Bug fix in `transaction_tests` (glozow) +- bitcoin/bitcoin#21293 Replace accidentally placed bit-OR with logical-OR (hebasto) +- bitcoin/bitcoin#21297 `feature_blockfilterindex_prune.py` improvements (jonatack) +- bitcoin/bitcoin#21310 zmq test: fix sync-up by matching notification to generated block (theStack) +- bitcoin/bitcoin#21334 Additional BIP9 tests (Sjors) +- bitcoin/bitcoin#21338 Add functional test for anchors.dat (brunoerg) +- bitcoin/bitcoin#21345 Bring `p2p_leak.py` up to date (mzumsande) +- bitcoin/bitcoin#21357 Unconditionally check for fRelay field in test framework (jarolrod) +- bitcoin/bitcoin#21358 fuzz: Add missing include (`test/util/setup_common.h`) (MarcoFalke) +- bitcoin/bitcoin#21371 fuzz: fix gcc Woverloaded-virtual build warnings (jonatack) +- bitcoin/bitcoin#21373 Generate fewer blocks in `feature_nulldummy` to fix timeouts, speed up (jonatack) +- bitcoin/bitcoin#21390 Test improvements for UTXO set hash tests (fjahr) +- bitcoin/bitcoin#21410 increase `rpc_timeout` for fundrawtx `test_transaction_too_large` (jonatack) +- bitcoin/bitcoin#21411 add logging, reduce blocks, move `sync_all` in `wallet_` groups (jonatack) +- bitcoin/bitcoin#21438 Add ParseUInt8() test coverage (jonatack) +- bitcoin/bitcoin#21443 fuzz: Implement `fuzzed_dns_lookup_function` as a lambda (practicalswift) +- bitcoin/bitcoin#21445 cirrus: Use SSD cluster for speedup (MarcoFalke) +- bitcoin/bitcoin#21477 Add test for CNetAddr::ToString IPv6 address formatting (RFC 5952) (practicalswift) +- bitcoin/bitcoin#21487 fuzz: Use ConsumeWeakEnum in addrman for service flags (MarcoFalke) +- bitcoin/bitcoin#21488 Add ParseUInt16() unit test and fuzz coverage (jonatack) +- bitcoin/bitcoin#21491 test: remove duplicate assertions in util_tests (jonatack) +- bitcoin/bitcoin#21522 fuzz: Use PickValue where possible (MarcoFalke) +- bitcoin/bitcoin#21531 remove qt byteswap compattests (fanquake) +- bitcoin/bitcoin#21557 small cleanup in RPCNestedTests tests (fanquake) +- bitcoin/bitcoin#21586 Add missing suppression for signed-integer-overflow:txmempool.cpp (MarcoFalke) +- bitcoin/bitcoin#21592 Remove option to make TestChain100Setup non-deterministic (MarcoFalke) +- bitcoin/bitcoin#21597 Document `race:validation_chainstatemanager_tests` suppression (MarcoFalke) +- bitcoin/bitcoin#21599 Replace file level integer overflow suppression with function level suppression (practicalswift) +- bitcoin/bitcoin#21604 Document why no symbol names can be used for suppressions (MarcoFalke) +- bitcoin/bitcoin#21606 fuzz: Extend psbt fuzz target a bit (MarcoFalke) +- bitcoin/bitcoin#21617 fuzz: Fix uninitialized read in i2p test (MarcoFalke) +- bitcoin/bitcoin#21630 fuzz: split FuzzedSock interface and implementation (vasild) +- bitcoin/bitcoin#21634 Skip SQLite fsyncs while testing (achow101) +- bitcoin/bitcoin#21669 Remove spurious double lock tsan suppressions by bumping to clang-12 (MarcoFalke) +- bitcoin/bitcoin#21676 Use mocktime to avoid intermittent failure in `rpc_tests` (MarcoFalke) +- bitcoin/bitcoin#21677 fuzz: Avoid use of low file descriptor ids (which may be in use) in FuzzedSock (practicalswift) +- bitcoin/bitcoin#21678 Fix TestPotentialDeadLockDetected suppression (hebasto) +- bitcoin/bitcoin#21689 Remove intermittently failing and not very meaningful `BOOST_CHECK` in `cnetaddr_basic` (practicalswift) +- bitcoin/bitcoin#21691 Check that no versionbits are re-used (MarcoFalke) +- bitcoin/bitcoin#21707 Extend functional tests for addr relay (mzumsande) +- bitcoin/bitcoin#21712 Test default `include_mempool` value of gettxout (promag) +- bitcoin/bitcoin#21738 Use clang-12 for ASAN, Add missing suppression (MarcoFalke) +- bitcoin/bitcoin#21740 add new python linter to check file names and permissions (windsok) +- bitcoin/bitcoin#21749 Bump shellcheck version (hebasto) +- bitcoin/bitcoin#21754 Run `feature_cltv` with MiniWallet (MarcoFalke) +- bitcoin/bitcoin#21762 Speed up `mempool_spend_coinbase.py` (MarcoFalke) +- bitcoin/bitcoin#21773 fuzz: Ensure prevout is consensus-valid (MarcoFalke) +- bitcoin/bitcoin#21777 Fix `feature_notifications.py` intermittent issue (MarcoFalke) +- bitcoin/bitcoin#21785 Fix intermittent issue in `p2p_addr_relay.py` (MarcoFalke) +- bitcoin/bitcoin#21787 Fix off-by-ones in `rpc_fundrawtransaction` assertions (jonatack) +- bitcoin/bitcoin#21792 Fix intermittent issue in `p2p_segwit.py` (MarcoFalke) +- bitcoin/bitcoin#21795 fuzz: Terminate immediately if a fuzzing harness tries to perform a DNS lookup (belt and suspenders) (practicalswift) +- bitcoin/bitcoin#21798 fuzz: Create a block template in `tx_pool` targets (MarcoFalke) +- bitcoin/bitcoin#21804 Speed up `p2p_segwit.py` (jnewbery) +- bitcoin/bitcoin#21810 fuzz: Various RPC fuzzer follow-ups (practicalswift) +- bitcoin/bitcoin#21814 Fix `feature_config_args.py` intermittent issue (MarcoFalke) +- bitcoin/bitcoin#21821 Add missing test for empty P2WSH redeem (MarcoFalke) +- bitcoin/bitcoin#21822 Resolve bug in `interface_bitcoin_cli.py` (klementtan) +- bitcoin/bitcoin#21846 fuzz: Add `-fsanitize=integer` suppression needed for RPC fuzzer (`generateblock`) (practicalswift) +- bitcoin/bitcoin#21849 fuzz: Limit toxic test globals to their respective scope (MarcoFalke) +- bitcoin/bitcoin#21867 use MiniWallet for `p2p_blocksonly.py` (theStack) +- bitcoin/bitcoin#21873 minor fixes & improvements for files linter test (windsok) +- bitcoin/bitcoin#21874 fuzz: Add `WRITE_ALL_FUZZ_TARGETS_AND_ABORT` (MarcoFalke) +- bitcoin/bitcoin#21884 fuzz: Remove unused --enable-danger-fuzz-link-all option (MarcoFalke) +- bitcoin/bitcoin#21890 fuzz: Limit ParseISO8601DateTime fuzzing to 32-bit (MarcoFalke) +- bitcoin/bitcoin#21891 fuzz: Remove strprintf test cases that are known to fail (MarcoFalke) +- bitcoin/bitcoin#21892 fuzz: Avoid excessively large min fee rate in `tx_pool` (MarcoFalke) +- bitcoin/bitcoin#21895 Add TSA annotations to the WorkQueue class members (hebasto) +- bitcoin/bitcoin#21900 use MiniWallet for `feature_csv_activation.py` (theStack) +- bitcoin/bitcoin#21909 fuzz: Limit max insertions in timedata fuzz test (MarcoFalke) +- bitcoin/bitcoin#21922 fuzz: Avoid timeout in EncodeBase58 (MarcoFalke) +- bitcoin/bitcoin#21927 fuzz: Run const CScript member functions only once (MarcoFalke) +- bitcoin/bitcoin#21929 fuzz: Remove incorrect float round-trip serialization test (MarcoFalke) +- bitcoin/bitcoin#21936 fuzz: Terminate immediately if a fuzzing harness tries to create a TCP socket (belt and suspenders) (practicalswift) +- bitcoin/bitcoin#21941 fuzz: Call const member functions in addrman fuzz test only once (MarcoFalke) +- bitcoin/bitcoin#21945 add P2PK support to MiniWallet (theStack) +- bitcoin/bitcoin#21948 Fix off-by-one in mockscheduler test RPC (MarcoFalke) +- bitcoin/bitcoin#21953 fuzz: Add `utxo_snapshot` target (MarcoFalke) +- bitcoin/bitcoin#21970 fuzz: Add missing CheckTransaction before CheckTxInputs (MarcoFalke) +- bitcoin/bitcoin#21989 Use `COINBASE_MATURITY` in functional tests (kiminuo) +- bitcoin/bitcoin#22003 Add thread safety annotations (ajtowns) +- bitcoin/bitcoin#22004 fuzz: Speed up transaction fuzz target (MarcoFalke) +- bitcoin/bitcoin#22005 fuzz: Speed up banman fuzz target (MarcoFalke) +- bitcoin/bitcoin#22029 [fuzz] Improve transport deserialization fuzz test coverage (dhruv) +- bitcoin/bitcoin#22048 MiniWallet: introduce enum type for output mode (theStack) +- bitcoin/bitcoin#22057 use MiniWallet (P2PK mode) for `feature_dersig.py` (theStack) +- bitcoin/bitcoin#22065 Mark `CheckTxInputs` `[[nodiscard]]`. Avoid UUM in fuzzing harness `coins_view` (practicalswift) +- bitcoin/bitcoin#22069 fuzz: don't try and use fopencookie() when building for Android (fanquake) +- bitcoin/bitcoin#22082 update nanobench from release 4.0.0 to 4.3.4 (martinus) +- bitcoin/bitcoin#22086 remove BasicTestingSetup from unit tests that don't need it (fanquake) +- bitcoin/bitcoin#22089 MiniWallet: fix fee calculation for P2PK and check tx vsize (theStack) +- bitcoin/bitcoin#21107, bitcoin/bitcoin#22092 Convert documentation into type annotations (fanquake) +- bitcoin/bitcoin#22095 Additional BIP32 test vector for hardened derivation with leading zeros (kristapsk) +- bitcoin/bitcoin#22103 Fix IPv6 check on BSD systems (n-thumann) +- bitcoin/bitcoin#22118 check anchors.dat when node starts for the first time (brunoerg) +- bitcoin/bitcoin#22120 `p2p_invalid_block`: Check that a block rejected due to too-new tim… (willcl-ark) +- bitcoin/bitcoin#22153 Fix `p2p_leak.py` intermittent failure (mzumsande) +- bitcoin/bitcoin#22169 p2p, rpc, fuzz: various tiny follow-ups (jonatack) +- bitcoin/bitcoin#22176 Correct outstanding -Werror=sign-compare errors (Empact) +- bitcoin/bitcoin#22180 fuzz: Increase branch coverage of the float fuzz target (MarcoFalke) +- bitcoin/bitcoin#22187 Add `sync_blocks` in `wallet_orphanedreward.py` (domob1812) +- bitcoin/bitcoin#22201 Fix TestShell to allow running in Jupyter Notebook (josibake) +- bitcoin/bitcoin#22202 Add temporary coinstats suppressions (MarcoFalke) +- bitcoin/bitcoin#22203 Use ConnmanTestMsg from test lib in `denialofservice_tests` (MarcoFalke) +- bitcoin/bitcoin#22210 Use MiniWallet in `test_no_inherited_signaling` RBF test (MarcoFalke) +- bitcoin/bitcoin#22224 Update msvc and appveyor builds to use Qt5.12.11 binaries (sipsorcery) +- bitcoin/bitcoin#22249 Kill process group to avoid dangling processes when using `--failfast` (S3RK) +- bitcoin/bitcoin#22267 fuzz: Speed up crypto fuzz target (MarcoFalke) +- bitcoin/bitcoin#22270 Add bitcoin-util tests (+refactors) (MarcoFalke) +- bitcoin/bitcoin#22271 fuzz: Assert roundtrip equality for `CPubKey` (theStack) +- bitcoin/bitcoin#22279 fuzz: add missing ECCVerifyHandle to `base_encode_decode` (apoelstra) +- bitcoin/bitcoin#22292 bench, doc: benchmarking updates and fixups (jonatack) +- bitcoin/bitcoin#22306 Improvements to `p2p_addr_relay.py` (amitiuttarwar) +- bitcoin/bitcoin#22310 Add functional test for replacement relay fee check (ariard) +- bitcoin/bitcoin#22311 Add missing syncwithvalidationinterfacequeue in `p2p_blockfilters` (MarcoFalke) +- bitcoin/bitcoin#22313 Add missing `sync_all` to `feature_coinstatsindex` (MarcoFalke) +- bitcoin/bitcoin#22322 fuzz: Check banman roundtrip (MarcoFalke) +- bitcoin/bitcoin#22363 Use `script_util` helpers for creating P2{PKH,SH,WPKH,WSH} scripts (theStack) +- bitcoin/bitcoin#22399 fuzz: Rework CTxDestination fuzzing (MarcoFalke) +- bitcoin/bitcoin#22408 add tests for `bad-txns-prevout-null` reject reason (theStack) +- bitcoin/bitcoin#22445 fuzz: Move implementations of non-template fuzz helpers from util.h to util.cpp (sriramdvt) +- bitcoin/bitcoin#22446 Fix `wallet_listdescriptors.py` if bdb is not compiled (hebasto) +- bitcoin/bitcoin#22447 Whitelist `rpc_rawtransaction` peers to speed up tests (jonatack) +- bitcoin/bitcoin#22742 Use proper target in `do_fund_send` (S3RK) + +### Miscellaneous +- bitcoin/bitcoin#19337 sync: Detect double lock from the same thread (vasild) +- bitcoin/bitcoin#19809 log: Prefix log messages with function name and source code location if -logsourcelocations is set (practicalswift) +- bitcoin/bitcoin#19866 eBPF Linux tracepoints (jb55) +- bitcoin/bitcoin#20024 init: Fix incorrect warning "Reducing -maxconnections from N to N-1, because of system limitations" (practicalswift) +- bitcoin/bitcoin#20145 contrib: Add getcoins.py script to get coins from (signet) faucet (kallewoof) +- bitcoin/bitcoin#20255 util: Add assume() identity function (MarcoFalke) +- bitcoin/bitcoin#20288 script, doc: Contrib/seeds updates (jonatack) +- bitcoin/bitcoin#20358 src/randomenv.cpp: Fix build on uclibc (ffontaine) +- bitcoin/bitcoin#20406 util: Avoid invalid integer negation in formatmoney and valuefromamount (practicalswift) +- bitcoin/bitcoin#20434 contrib: Parse elf directly for symbol and security checks (laanwj) +- bitcoin/bitcoin#20451 lint: Run mypy over contrib/devtools (fanquake) +- bitcoin/bitcoin#20476 contrib: Add test for elf symbol-check (laanwj) +- bitcoin/bitcoin#20530 lint: Update cppcheck linter to c++17 and improve explicit usage (fjahr) +- bitcoin/bitcoin#20589 log: Clarify that failure to read/write `fee_estimates.dat` is non-fatal (MarcoFalke) +- bitcoin/bitcoin#20602 util: Allow use of c++14 chrono literals (MarcoFalke) +- bitcoin/bitcoin#20605 init: Signal-safe instant shutdown (laanwj) +- bitcoin/bitcoin#20608 contrib: Add symbol check test for PE binaries (fanquake) +- bitcoin/bitcoin#20689 contrib: Replace binary verification script verify.sh with python rewrite (theStack) +- bitcoin/bitcoin#20715 util: Add argsmanager::getcommand() and use it in bitcoin-wallet (MarcoFalke) +- bitcoin/bitcoin#20735 script: Remove outdated extract-osx-sdk.sh (hebasto) +- bitcoin/bitcoin#20817 lint: Update list of spelling linter false positives, bump to codespell 2.0.0 (theStack) +- bitcoin/bitcoin#20884 script: Improve robustness of bitcoind.service on startup (hebasto) +- bitcoin/bitcoin#20906 contrib: Embed c++11 patch in `install_db4.sh` (gruve-p) +- bitcoin/bitcoin#21004 contrib: Fix docker args conditional in gitian-build (setpill) +- bitcoin/bitcoin#21007 bitcoind: Add -daemonwait option to wait for initialization (laanwj) +- bitcoin/bitcoin#21041 log: Move "Pre-allocating up to position 0x[…] in […].dat" log message to debug category (practicalswift) +- bitcoin/bitcoin#21059 Drop boost/preprocessor dependencies (hebasto) +- bitcoin/bitcoin#21087 guix: Passthrough `BASE_CACHE` into container (dongcarl) +- bitcoin/bitcoin#21088 guix: Jump forwards in time-machine and adapt (dongcarl) +- bitcoin/bitcoin#21089 guix: Add support for powerpc64{,le} (dongcarl) +- bitcoin/bitcoin#21110 util: Remove boost `posix_time` usage from `gettime*` (fanquake) +- bitcoin/bitcoin#21111 Improve OpenRC initscript (parazyd) +- bitcoin/bitcoin#21123 code style: Add EditorConfig file (kiminuo) +- bitcoin/bitcoin#21173 util: Faster hexstr => 13% faster blocktojson (martinus) +- bitcoin/bitcoin#21221 tools: Allow argument/parameter bin packing in clang-format (jnewbery) +- bitcoin/bitcoin#21244 Move GetDataDir to ArgsManager (kiminuo) +- bitcoin/bitcoin#21255 contrib: Run test-symbol-check for risc-v (fanquake) +- bitcoin/bitcoin#21271 guix: Explicitly set umask in build container (dongcarl) +- bitcoin/bitcoin#21300 script: Add explanatory comment to tc.sh (dscotese) +- bitcoin/bitcoin#21317 util: Make assume() usable as unary expression (MarcoFalke) +- bitcoin/bitcoin#21336 Make .gitignore ignore src/test/fuzz/fuzz.exe (hebasto) +- bitcoin/bitcoin#21337 guix: Update darwin native packages dependencies (hebasto) +- bitcoin/bitcoin#21405 compat: remove memcpy -> memmove backwards compatibility alias (fanquake) +- bitcoin/bitcoin#21418 contrib: Make systemd invoke dependencies only when ready (laanwj) +- bitcoin/bitcoin#21447 Always add -daemonwait to known command line arguments (hebasto) +- bitcoin/bitcoin#21471 bugfix: Fix `bech32_encode` calls in `gen_key_io_test_vectors.py` (sipa) +- bitcoin/bitcoin#21615 script: Add trusted key for hebasto (hebasto) +- bitcoin/bitcoin#21664 contrib: Use lief for macos and windows symbol & security checks (fanquake) +- bitcoin/bitcoin#21695 contrib: Remove no longer used contrib/bitcoin-qt.pro (hebasto) +- bitcoin/bitcoin#21711 guix: Add full installation and usage documentation (dongcarl) +- bitcoin/bitcoin#21799 guix: Use `gcc-8` across the board (dongcarl) +- bitcoin/bitcoin#21802 Avoid UB in util/asmap (advance a dereferenceable iterator outside its valid range) (MarcoFalke) +- bitcoin/bitcoin#21823 script: Update reviewers (jonatack) +- bitcoin/bitcoin#21850 Remove `GetDataDir(net_specific)` function (kiminuo) +- bitcoin/bitcoin#21871 scripts: Add checks for minimum required os versions (fanquake) +- bitcoin/bitcoin#21966 Remove double serialization; use software encoder for fee estimation (sipa) +- bitcoin/bitcoin#22060 contrib: Add torv3 seed nodes for testnet, drop v2 ones (laanwj) +- bitcoin/bitcoin#22244 devtools: Correctly extract symbol versions in symbol-check (laanwj) +- bitcoin/bitcoin#22533 guix/build: Remove vestigial SKIPATTEST.TAG (dongcarl) +- bitcoin/bitcoin#22643 guix-verify: Non-zero exit code when anything fails (dongcarl) +- bitcoin/bitcoin#22654 guix: Don't include directory name in SHA256SUMS (achow101) + +### Documentation +- bitcoin/bitcoin#15451 clarify getdata limit after #14897 (HashUnlimited) +- bitcoin/bitcoin#15545 Explain why CheckBlock() is called before AcceptBlock (Sjors) +- bitcoin/bitcoin#17350 Add developer documentation to isminetype (HAOYUatHZ) +- bitcoin/bitcoin#17934 Use `CONFIG_SITE` variable instead of --prefix option (hebasto) +- bitcoin/bitcoin#18030 Coin::IsSpent() can also mean never existed (Sjors) +- bitcoin/bitcoin#18096 IsFinalTx comment about nSequence & `OP_CLTV` (nothingmuch) +- bitcoin/bitcoin#18568 Clarify developer notes about constant naming (ryanofsky) +- bitcoin/bitcoin#19961 doc: tor.md updates (jonatack) +- bitcoin/bitcoin#19968 Clarify CRollingBloomFilter size estimate (robot-dreams) +- bitcoin/bitcoin#20200 Rename CODEOWNERS to REVIEWERS (adamjonas) +- bitcoin/bitcoin#20329 docs/descriptors.md: Remove hardened marker in the path after xpub (dgpv) +- bitcoin/bitcoin#20380 Add instructions on how to fuzz the P2P layer using Honggfuzz NetDriver (practicalswift) +- bitcoin/bitcoin#20414 Remove generated manual pages from master branch (laanwj) +- bitcoin/bitcoin#20473 Document current boost dependency as 1.71.0 (laanwj) +- bitcoin/bitcoin#20512 Add bash as an OpenBSD dependency (emilengler) +- bitcoin/bitcoin#20568 Use FeeModes doc helper in estimatesmartfee (MarcoFalke) +- bitcoin/bitcoin#20577 libconsensus: add missing error code description, fix NBitcoin link (theStack) +- bitcoin/bitcoin#20587 Tidy up Tor doc (more stringent) (wodry) +- bitcoin/bitcoin#20592 Update wtxidrelay documentation per BIP339 (jonatack) +- bitcoin/bitcoin#20601 Update for FreeBSD 12.2, add GUI Build Instructions (jarolrod) +- bitcoin/bitcoin#20635 fix misleading comment about call to non-existing function (pox) +- bitcoin/bitcoin#20646 Refer to BIPs 339/155 in feature negotiation (jonatack) +- bitcoin/bitcoin#20653 Move addr relay comment in net to correct place (MarcoFalke) +- bitcoin/bitcoin#20677 Remove shouty enums in `net_processing` comments (sdaftuar) +- bitcoin/bitcoin#20741 Update 'Secure string handling' (prayank23) +- bitcoin/bitcoin#20757 tor.md and -onlynet help updates (jonatack) +- bitcoin/bitcoin#20829 Add -netinfo help (jonatack) +- bitcoin/bitcoin#20830 Update developer notes with signet (jonatack) +- bitcoin/bitcoin#20890 Add explicit macdeployqtplus dependencies install step (hebasto) +- bitcoin/bitcoin#20913 Add manual page generation for bitcoin-util (laanwj) +- bitcoin/bitcoin#20985 Add xorriso to macOS depends packages (fanquake) +- bitcoin/bitcoin#20986 Update developer notes to discourage very long lines (jnewbery) +- bitcoin/bitcoin#20987 Add instructions for generating RPC docs (ben-kaufman) +- bitcoin/bitcoin#21026 Document use of make-tag script to make tags (laanwj) +- bitcoin/bitcoin#21028 doc/bips: Add BIPs 43, 44, 49, and 84 (luke-jr) +- bitcoin/bitcoin#21049 Add release notes for listdescriptors RPC (S3RK) +- bitcoin/bitcoin#21060 More precise -debug and -debugexclude doc (wodry) +- bitcoin/bitcoin#21077 Clarify -timeout and -peertimeout config options (glozow) +- bitcoin/bitcoin#21105 Correctly identify script type (niftynei) +- bitcoin/bitcoin#21163 Guix is shipped in Debian and Ubuntu (MarcoFalke) +- bitcoin/bitcoin#21210 Rework internal and external links (MarcoFalke) +- bitcoin/bitcoin#21246 Correction for VerifyTaprootCommitment comments (roconnor-blockstream) +- bitcoin/bitcoin#21263 Clarify that squashing should happen before review (MarcoFalke) +- bitcoin/bitcoin#21323 guix, doc: Update default HOSTS value (hebasto) +- bitcoin/bitcoin#21324 Update build instructions for Fedora (hebasto) +- bitcoin/bitcoin#21343 Revamp macOS build doc (jarolrod) +- bitcoin/bitcoin#21346 install qt5 when building on macOS (fanquake) +- bitcoin/bitcoin#21384 doc: add signet to bitcoin.conf documentation (jonatack) +- bitcoin/bitcoin#21394 Improve comment about protected peers (amitiuttarwar) +- bitcoin/bitcoin#21398 Update fuzzing docs for afl-clang-lto (MarcoFalke) +- bitcoin/bitcoin#21444 net, doc: Doxygen updates and fixes in netbase.{h,cpp} (jonatack) +- bitcoin/bitcoin#21481 Tell howto install clang-format on Debian/Ubuntu (wodry) +- bitcoin/bitcoin#21567 Fix various misleading comments (glozow) +- bitcoin/bitcoin#21661 Fix name of script guix-build (Emzy) +- bitcoin/bitcoin#21672 Remove boostrap info from `GUIX_COMMON_FLAGS` doc (fanquake) +- bitcoin/bitcoin#21688 Note on SDK for macOS depends cross-compile (jarolrod) +- bitcoin/bitcoin#21709 Update reduce-memory.md and bitcoin.conf -maxconnections info (jonatack) +- bitcoin/bitcoin#21710 update helps for addnode rpc and -addnode/-maxconnections config options (jonatack) +- bitcoin/bitcoin#21752 Clarify that feerates are per virtual size (MarcoFalke) +- bitcoin/bitcoin#21811 Remove Visual Studio 2017 reference from readme (sipsorcery) +- bitcoin/bitcoin#21818 Fixup -coinstatsindex help, update bitcoin.conf and files.md (jonatack) +- bitcoin/bitcoin#21856 add OSS-Fuzz section to fuzzing.md doc (adamjonas) +- bitcoin/bitcoin#21912 Remove mention of priority estimation (MarcoFalke) +- bitcoin/bitcoin#21925 Update bips.md for 0.21.1 (MarcoFalke) +- bitcoin/bitcoin#21942 improve make with parallel jobs description (klementtan) +- bitcoin/bitcoin#21947 Fix OSS-Fuzz links (MarcoFalke) +- bitcoin/bitcoin#21988 note that brew installed qt is not supported (jarolrod) +- bitcoin/bitcoin#22056 describe in fuzzing.md how to reproduce a CI crash (jonatack) +- bitcoin/bitcoin#22080 add maxuploadtarget to bitcoin.conf example (jarolrod) +- bitcoin/bitcoin#22088 Improve note on choosing posix mingw32 (jarolrod) +- bitcoin/bitcoin#22109 Fix external links (IRC, …) (MarcoFalke) +- bitcoin/bitcoin#22121 Various validation doc fixups (MarcoFalke) +- bitcoin/bitcoin#22172 Update tor.md, release notes with removal of tor v2 support (jonatack) +- bitcoin/bitcoin#22204 Remove obsolete `okSafeMode` RPC guideline from developer notes (theStack) +- bitcoin/bitcoin#22208 Update `REVIEWERS` (practicalswift) +- bitcoin/bitcoin#22250 add basic I2P documentation (vasild) +- bitcoin/bitcoin#22296 Final merge of release notes snippets, mv to wiki (MarcoFalke) +- bitcoin/bitcoin#22335 recommend `--disable-external-signer` in OpenBSD build guide (theStack) +- bitcoin/bitcoin#22339 Document minimum required libc++ version (hebasto) +- bitcoin/bitcoin#22349 Repository IRC updates (jonatack) +- bitcoin/bitcoin#22360 Remove unused section from release process (MarcoFalke) +- bitcoin/bitcoin#22369 Add steps for Transifex to release process (jonatack) +- bitcoin/bitcoin#22393 Added info to bitcoin.conf doc (bliotti) +- bitcoin/bitcoin#22402 Install Rosetta on M1-macOS for qt in depends (hebasto) +- bitcoin/bitcoin#22432 Fix incorrect `testmempoolaccept` doc (glozow) +- bitcoin/bitcoin#22648 doc, test: improve i2p/tor docs and i2p reachable unit tests (jonatack) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Aaron Clauson +- Adam Jonas +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Poelstra +- Anthony Towns +- Antoine Poinsot +- Antoine Riard +- apawlik +- apitko +- Ben Carman +- Ben Woosley +- benk10 +- Bezdrighin +- Block Mechanic +- Brian Liotti +- Bruno Garcia +- Carl Dong +- Christian Decker +- coinforensics +- Cory Fields +- Dan Benjamin +- Daniel Kraft +- Darius Parvin +- Dhruv Mehta +- Dmitry Goncharov +- Dmitry Petukhov +- dplusplus1024 +- dscotese +- Duncan Dean +- Elle Mouton +- Elliott Jin +- Emil Engler +- Ethan Heilman +- eugene +- Evan Klitzke +- Fabian Jahr +- Fabrice Fontaine +- fanquake +- fdov +- flack +- Fotis Koutoupas +- Fu Yong Quah +- fyquah +- glozow +- Gregory Sanders +- Guido Vranken +- Gunar C. Gessner +- h +- HAOYUatHZ +- Hennadii Stepanov +- Igor Cota +- Ikko Ashimine +- Ivan Metlushko +- jackielove4u +- James O'Beirne +- Jarol Rodriguez +- Joel Klabo +- John Newbery +- Jon Atack +- Jonas Schnelli +- João Barbosa +- Josiah Baker +- Karl-Johan Alm +- Kiminuo +- Klement Tan +- Kristaps Kaupe +- Larry Ruane +- lisa neigut +- Lucas Ontivero +- Luke Dashjr +- Maayan Keshet +- MarcoFalke +- Martin Ankerl +- Martin Zumsande +- Michael Dietz +- Michael Polzer +- Michael Tidwell +- Niklas Gögge +- nthumann +- Oliver Gugger +- parazyd +- Patrick Strateman +- Pavol Rusnak +- Peter Bushnell +- Pierre K +- Pieter Wuille +- PiRK +- pox +- practicalswift +- Prayank +- R E Broadley +- Rafael Sadowski +- randymcmillan +- Raul Siles +- Riccardo Spagni +- Russell O'Connor +- Russell Yanofsky +- S3RK +- saibato +- Samuel Dobson +- sanket1729 +- Sawyer Billings +- Sebastian Falbesoner +- setpill +- sgulls +- sinetek +- Sjors Provoost +- Sriram +- Stephan Oeste +- Suhas Daftuar +- Sylvain Goumy +- t-bast +- Troy Giorshev +- Tushar Singla +- Tyler Chambers +- Uplab +- Vasil Dimov +- W. J. van der Laan +- willcl-ark +- William Bright +- William Casarin +- windsok +- wodry +- Yerzhan Mazhkenov +- Yuval Kogman +- Zero + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-22.1.md b/doc/release-notes/release-notes-22.1.md new file mode 100644 index 000000000000..d304b7e57aef --- /dev/null +++ b/doc/release-notes/release-notes-22.1.md @@ -0,0 +1,128 @@ +22.1 Release Notes +================== + +Bitcoin Core version 22.1 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 22.0 onwards, macOS versions earlier than 10.14 are no longer supported. + +Notable changes +=============== + +Updated settings +---------------- + +- In previous releases, the meaning of the command line option + `-persistmempool` (without a value provided) incorrectly disabled mempool + persistence. `-persistmempool` is now treated like other boolean options to + mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` + and `-nopersistmempool` is unaffected. (#23061) + +### P2P + +### RPC and other APIs + +- #25237 rpc: Capture UniValue by ref for rpcdoccheck +- #25983 Prevent data race for pathHandlers +- #26275 Fix crash on deriveaddresses when index is 2147483647 (2^31-1) + +### Wallet + +- #22781 wallet: fix the behavior of IsHDEnabled +- #22949 fee: Round up fee calculation to avoid a lower than expected feerate +- #23333 wallet: fix segfault by avoiding invalid default-ctored external_spk_managers entry + +### Build system + +- #22820 build, qt: Fix typo in QtInputSupport check +- #23045 build: Restrict check for CRC32C intrinsic to aarch64 +- #23148 build: Fix guix linker-loader path and add check_ELF_interpreter +- #23314 build: explicitly disable libsecp256k1 openssl based tests +- #23580 build: patch qt to explicitly define previously implicit header include +- #24215 guix: ignore additional failing certvalidator test +- #24256 build: Bump depends packages (zmq, libXau) +- #25201 windeploy: Renewed windows code signing certificate +- #25985 Revert "build: Use Homebrew's sqlite package if it is available" +- #26633 depends: update qt 5.12 url to archive location + +### GUI + +- #gui631 Disallow encryption of watchonly wallets +- #gui680 Fixes MacOS 13 segfault by preventing certain notifications +- #24498 qt: Avoid crash on startup if int specified in settings.json + +### Tests + +- #23716 test: replace hashlib.ripemd160 with an own implementation +- #24239 test: fix ceildiv division by using integers + +### Utilities + +- #22390 system: skip trying to set the locale on NetBSD +- #22895 don't call GetBlockPos in ReadBlockFromDisk without cs_main lock +- #24104 fs: Make compatible with boost 1.78 + +### Miscellaneous + +- #23335 refactor: include a missing header in fs.cpp +- #23504 ci: Replace soon EOL hirsute with jammy +- #26321 Adjust .tx/config for new Transifex CLI + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- BlackcoinDev +- Carl Dong +- Hennadii Stepanov +- Joan Karadimov +- John Moffett +- Jon Atack +- Kittywhiskers Van Gogh +- Marco Falke +- Martin Zumsande +- Michael Ford +- muxator +- Pieter Wuille +- Ryan Ofsky +- Saibato +- Sebastian Falbesoner +- W. J. van der Laan + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-23.0.md b/doc/release-notes/release-notes-23.0.md new file mode 100644 index 000000000000..b1467a0f7115 --- /dev/null +++ b/doc/release-notes/release-notes-23.0.md @@ -0,0 +1,373 @@ +23.0 Release Notes +================== + +Bitcoin Core version 23.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +- A bitcoind node will no longer rumour addresses to inbound peers by default. + They will become eligible for address gossip after sending an ADDR, ADDRV2, + or GETADDR message. (#21528) + +- Before this release, Bitcoin Core had a strong preference to try to connect only to peers that listen on port 8333. As a result of that, Bitcoin nodes listening on non-standard ports would likely not get any Bitcoin Core peers connecting to them. This preference has been removed. (#23542) + +- Full support has been added for the CJDNS network. See the new option `-cjdnsreachable` and [doc/cjdns.md](https://github.com/bitcoin/bitcoin/tree/23.x/doc/cjdns.md) (#23077) + +Fee estimation changes +---------------------- + +- Fee estimation now takes the feerate of replacement (RBF) transactions into + account. (#22539) + +Rescan startup parameter removed +-------------------------------- + +The `-rescan` startup parameter has been removed. Wallets which require +rescanning due to corruption will still be rescanned on startup. +Otherwise, please use the `rescanblockchain` RPC to trigger a rescan. (#23123) + +Tracepoints and Userspace, Statically Defined Tracing support +------------------------------------------------------------- + +Bitcoin Core release binaries for Linux now include experimental tracepoints which +act as an interface for process-internal events. These can be used for review, +debugging, monitoring, and more. The tracepoint API is semi-stable. While the API +is tested, process internals might change between releases requiring changes to the +tracepoints. Information about the existing tracepoints can be found under +[doc/tracing.md](https://github.com/bitcoin/bitcoin/blob/23.x/doc/tracing.md) and +usage examples are provided in [contrib/tracing/](https://github.com/bitcoin/bitcoin/tree/23.x/contrib/tracing). + +Updated RPCs +------------ + +- The `validateaddress` RPC now returns an `error_locations` array for invalid + addresses, with the indices of invalid character locations in the address (if + known). For example, this will attempt to locate up to two Bech32 errors, and + return their locations if successful. Success and correctness are only guaranteed + if fewer than two substitution errors have been made. + The error message returned in the `error` field now also returns more specific + errors when decoding fails. (#16807) + +- The `-deprecatedrpc=addresses` configuration option has been removed. RPCs + `gettxout`, `getrawtransaction`, `decoderawtransaction`, `decodescript`, + `gettransaction verbose=true` and REST endpoints `/rest/tx`, `/rest/getutxos`, + `/rest/block` no longer return the `addresses` and `reqSigs` fields, which + were previously deprecated in 22.0. (#22650) +- The `getblock` RPC command now supports verbosity level 3 containing transaction inputs' + `prevout` information. The existing `/rest/block/` REST endpoint is modified to contain + this information too. Every `vin` field will contain an additional `prevout` subfield + describing the spent output. `prevout` contains the following keys: + - `generated` - true if the spent coins was a coinbase. + - `height` + - `value` + - `scriptPubKey` + +- The top-level fee fields `fee`, `modifiedfee`, `ancestorfees` and `descendantfees` + returned by RPCs `getmempoolentry`,`getrawmempool(verbose=true)`, + `getmempoolancestors(verbose=true)` and `getmempooldescendants(verbose=true)` + are deprecated and will be removed in the next major version (use + `-deprecated=fees` if needed in this version). The same fee fields can be accessed + through the `fees` object in the result. WARNING: deprecated + fields `ancestorfees` and `descendantfees` are denominated in sats, whereas all + fields in the `fees` object are denominated in BTC. (#22689) + +- Both `createmultisig` and `addmultisigaddress` now include a `warnings` + field, which will show a warning if a non-legacy address type is requested + when using uncompressed public keys. (#23113) + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +- Information on soft fork status has been moved from `getblockchaininfo` + to the new `getdeploymentinfo` RPC which allows querying soft fork status at any + block, rather than just at the chain tip. Inclusion of soft fork + status in `getblockchaininfo` can currently be restored using the + configuration `-deprecatedrpc=softforks`, but this will be removed in + a future release. Note that in either case, the `status` field + now reflects the status of the current block rather than the next + block. (#23508) + +Files +----- + +* On startup, the list of banned hosts and networks (via `setban` RPC) in + `banlist.dat` is ignored and only `banlist.json` is considered. Bitcoin Core + version 22.x is the only version that can read `banlist.dat` and also write + it to `banlist.json`. If `banlist.json` already exists, version 22.x will not + try to translate the `banlist.dat` into json. After an upgrade, `listbanned` + can be used to double check the parsed entries. (#22570) + +Updated settings +---------------- + +- In previous releases, the meaning of the command line option + `-persistmempool` (without a value provided) incorrectly disabled mempool + persistence. `-persistmempool` is now treated like other boolean options to + mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` + and `-nopersistmempool` is unaffected. (#23061) + +- `-maxuploadtarget` now allows human readable byte units [k|K|m|M|g|G|t|T]. + E.g. `-maxuploadtarget=500g`. No whitespace, +- or fractions allowed. + Default is `M` if no suffix provided. (#23249) + +- If `-proxy=` is given together with `-noonion` then the provided proxy will + not be set as a proxy for reaching the Tor network. So it will not be + possible to open manual connections to the Tor network for example with the + `addnode` RPC. To mimic the old behavior use `-proxy=` together with + `-onlynet=` listing all relevant networks except `onion`. (#22834) + +Tools and Utilities +------------------- + +- Update `-getinfo` to return data in a user-friendly format that also reduces vertical space. (#21832) + +- CLI `-addrinfo` now returns a single field for the number of `onion` addresses + known to the node instead of separate `torv2` and `torv3` fields, as support + for Tor V2 addresses was removed from Bitcoin Core in 22.0. (#22544) + +Wallet +------ + +- Descriptor wallets are now the default wallet type. Newly created wallets + will use descriptors unless `descriptors=false` is set during `createwallet`, or + the `Descriptor wallet` checkbox is unchecked in the GUI. + + Note that wallet RPC commands like `importmulti` and `dumpprivkey` cannot be + used with descriptor wallets, so if your client code relies on these commands + without specifying `descriptors=false` during wallet creation, you will need + to update your code. + +- Newly created descriptor wallets will contain an automatically generated `tr()` + descriptor which allows for creating single key Taproot receiving addresses. + +- `upgradewallet` will now automatically flush the keypool if upgrading + from a non-HD wallet to an HD wallet, to immediately start using the + newly-generated HD keys. (#23093) + +- a new RPC `newkeypool` has been added, which will flush (entirely + clear and refill) the keypool. (#23093) + +- `listunspent` now includes `ancestorcount`, `ancestorsize`, and + `ancestorfees` for each transaction output that is still in the mempool. + (#12677) + +- `lockunspent` now optionally takes a third parameter, `persistent`, which + causes the lock to be written persistently to the wallet database. This + allows UTXOs to remain locked even after node restarts or crashes. (#23065) + +- `receivedby` RPCs now include coinbase transactions. Previously, the + following wallet RPCs excluded coinbase transactions: `getreceivedbyaddress`, + `getreceivedbylabel`, `listreceivedbyaddress`, `listreceivedbylabel`. This + release changes this behaviour and returns results accounting for received + coins from coinbase outputs. The previous behaviour can be restored using the + configuration `-deprecatedrpc=exclude_coinbase`, but may be removed in a + future release. (#14707) + +- A new option in the same `receivedby` RPCs, `include_immature_coinbase` + (default=`false`), determines whether to account for immature coinbase + transactions. Immature coinbase transactions are coinbase transactions that + have 100 or fewer confirmations, and are not spendable. (#14707) + +GUI changes +----------- + +- UTXOs which are locked via the GUI are now stored persistently in the + wallet database, so are not lost on node shutdown or crash. (#23065) + +- The Bech32 checkbox has been replaced with a dropdown for all address types, including the new Bech32m (BIP-350) standard for Taproot enabled wallets. + +Low-level changes +================= + +RPC +--- + +- `getblockchaininfo` now returns a new `time` field, that provides the chain tip time. (#22407) + +Tests +----- + +- For the `regtest` network the activation heights of several softforks were + set to block height 1. They can be changed by the runtime setting + `-testactivationheight=name@height`. (#22818) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 0xree +- Aaron Clauson +- Adrian-Stefan Mares +- agroce +- aitorjs +- Alex Groce +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Poelstra +- Andrew Toth +- anouar kappitou +- Anthony Towns +- Antoine Poinsot +- Arnab Sen +- Ben Woosley +- benthecarman +- Bitcoin Hodler +- BitcoinTsunami +- brianddk +- Bruno Garcia +- CallMeMisterOwl +- Calvin Kim +- Carl Dong +- Cory Fields +- Cuong V. Nguyen +- Darius Parvin +- Dhruv Mehta +- Dimitri Deijs +- Dimitris Apostolou +- Dmitry Goncharov +- Douglas Chimento +- eugene +- Fabian Jahr +- fanquake +- Florian Baumgartl +- fyquah +- Gleb Naumenko +- glozow +- Gregory Sanders +- Heebs +- Hennadii Stepanov +- hg333 +- HiLivin +- Igor Cota +- Jadi +- James O'Beirne +- Jameson Lopp +- Jarol Rodriguez +- Jeremy Rand +- Jeremy Rubin +- Joan Karadimov +- John Newbery +- Jon Atack +- João Barbosa +- josibake +- junderw +- Karl-Johan Alm +- katesalazar +- Kennan Mell +- Kiminuo +- Kittywhiskers Van Gogh +- Klement Tan +- Kristaps Kaupe +- Kuro +- Larry Ruane +- lsilva01 +- lucash-dev +- Luke Dashjr +- MarcoFalke +- Martin Leitner-Ankerl +- Martin Zumsande +- Matt Corallo +- Matt Whitlock +- MeshCollider +- Michael Dietz +- Murch +- naiza +- Nathan Garabedian +- Nelson Galdeman +- NikhilBartwal +- Niklas Gögge +- node01 +- nthumann +- Pasta +- Patrick Kamin +- Pavel Safronov +- Pavol Rusnak +- Perlover +- Pieter Wuille +- practicalswift +- pradumnasaraf +- pranabp-bit +- Prateek Sancheti +- Prayank +- Rafael Sadowski +- rajarshimaitra +- randymcmillan +- ritickgoenka +- Rob Fielding +- Rojar Smith +- Russell Yanofsky +- S3RK +- Saibato +- Samuel Dobson +- sanket1729 +- seaona +- Sebastian Falbesoner +- sh15h4nk +- Shashwat +- Shorya +- ShubhamPalriwala +- Shubhankar Gambhir +- Sjors Provoost +- sogoagain +- sstone +- stratospher +- Suriyaa Rocky Sundararuban +- Taeik Lim +- TheCharlatan +- Tim Ruffing +- Tobin Harding +- Troy Giorshev +- Tyler Chambers +- Vasil Dimov +- W. J. van der Laan +- w0xlt +- willcl-ark +- William Casarin +- zealsham +- Zero-1729 + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-23.1.md b/doc/release-notes/release-notes-23.1.md new file mode 100644 index 000000000000..31d9b7f068d8 --- /dev/null +++ b/doc/release-notes/release-notes-23.1.md @@ -0,0 +1,90 @@ +23.1 Release Notes +================== + +Bitcoin Core version 23.1 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### P2P + +- #25314 p2p: always set nTime for self-advertisements + +### RPC and other APIs + +- #25220 rpc: fix incorrect warning for address type p2sh-segwit in createmultisig +- #25237 rpc: Capture UniValue by ref for rpcdoccheck +- #25983 Prevent data race for pathHandlers +- #26275 Fix crash on deriveaddresses when index is 2147483647 (2^31-1) + +### Build system + +- #25201 windeploy: Renewed windows code signing certificate +- #25788 guix: patch NSIS to remove .reloc sections from installer stubs +- #25861 guix: use --build={arch}-guix-linux-gnu in cross toolchain +- #25985 Revert "build: Use Homebrew's sqlite package if it is available" + +### GUI + +- #24668 build, qt: bump Qt5 version to 5.15.3 +- gui#631 Disallow encryption of watchonly wallets +- gui#680 Fixes MacOS 13 segfault by preventing certain notifications + +### Tests + +- #24454 tests: Fix calculation of external input weights + +### Miscellaneous + +- #26321 Adjust .tx/config for new Transifex CLI + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- brunoerg +- Hennadii Stepanov +- John Moffett +- MacroFake +- Martin Zumsande +- Michael Ford +- muxator +- Pavol Rusnak +- Sebastian Falbesoner +- W. J. van der Laan + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-24.0.1.md b/doc/release-notes/release-notes-24.0.1.md new file mode 100644 index 000000000000..24920ba4507e --- /dev/null +++ b/doc/release-notes/release-notes-24.0.1.md @@ -0,0 +1,391 @@ +24.0.1 Release Notes +==================== + +Due to last-minute issues (#26616), 24.0, although tagged, was never fully +announced or released. + +Bitcoin Core version 24.0.1 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notice of new option for transaction replacement policies +========================================================= + +This version of Bitcoin Core adds a new `mempoolfullrbf` configuration +option which allows users to change the policy their individual node +will use for relaying and mining unconfirmed transactions. The option +defaults to the same policy that was used in previous releases and no +changes to node policy will occur if everyone uses the default. + +Some Bitcoin services today expect that the first version of an +unconfirmed transaction that they see will be the version of the +transaction that ultimately gets confirmed---a transaction acceptance +policy sometimes called "first-seen". + +The Bitcoin Protocol does not, and cannot, provide any assurance that +the first version of an unconfirmed transaction seen by a particular +node will be the version that gets confirmed. If there are multiple +versions of the same unconfirmed transaction available, only the miner +who includes one of those transactions in a block gets to decide which +version of the transaction gets confirmed. + +Despite this lack of assurance, multiple merchants and services today +still make this assumption. + +There are several benefits to users from removing this *first-seen* +simplification. One key benefit, the ability for the sender of a +transaction to replace it with an alternative version paying higher +fees, was realized in [Bitcoin Core 0.12.0][] (February 2016) with the +introduction of [BIP125][] opt-in Replace By Fee (RBF). + +Since then, there has been discussion about completely removing the +first-seen simplification and allowing users to replace any of their +older unconfirmed transactions with newer transactions, a feature called +*full-RBF*. This release includes a `mempoolfullrbf` configuration +option that allows enabling full-RBF, although it defaults to off +(allowing only opt-in RBF). + +Several alternative node implementations have already enabled full-RBF by +default for years, and several contributors to Bitcoin Core are +advocating for enabling full-RBF by default in a future version of +Bitcoin Core. + +As more nodes that participate in relay and mining begin enabling +full-RBF, replacement of unconfirmed transactions by ones offering higher +fees may rapidly become more reliable. + +Contributors to this project strongly recommend that merchants and services +not accept unconfirmed transactions as final, and if they insist on doing so, +to take the appropriate steps to ensure they have some recourse or plan for +when their assumptions do not hold. + +[Bitcoin Core 0.12.0]: https://bitcoincore.org/en/releases/0.12.0/#opt-in-replace-by-fee-transactions +[bip125]: https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki + +Notable changes +=============== + +P2P and network changes +----------------------- + +- To address a potential denial-of-service, the logic to download headers from peers + has been reworked. This is particularly relevant for nodes starting up for the + first time (or for nodes which are starting up after being offline for a long time). + + Whenever headers are received from a peer that have a total chainwork that is either + less than the node's `-minimumchainwork` value or is sufficiently below the work at + the node's tip, a "presync" phase will begin, in which the node will download the + peer's headers and verify the cumulative work on the peer's chain, prior to storing + those headers permanently. Once that cumulative work is verified to be sufficiently high, + the headers will be redownloaded from that peer and fully validated and stored. + + This may result in initial headers sync taking longer for new nodes starting up for + the first time, both because the headers will be downloaded twice, and because the effect + of a peer disconnecting during the presync phase (or while the node's best headers chain has less + than `-minimumchainwork`), will result in the node needing to use the headers presync mechanism + with the next peer as well (downloading the headers twice, again). (#25717) + +- With I2P connections, a new, transient address is used for each outbound + connection if `-i2pacceptincoming=0`. (#25355) + +Updated RPCs +------------ + +- The `-deprecatedrpc=softforks` configuration option has been removed. The + RPC `getblockchaininfo` no longer returns the `softforks` field, which was + previously deprecated in 23.0. (#23508) Information on soft fork status is + now only available via the `getdeploymentinfo` RPC. + +- The `deprecatedrpc=exclude_coinbase` configuration option has been removed. + The `receivedby` RPCs (`listreceivedbyaddress`, `listreceivedbylabel`, + `getreceivedbyaddress` and `getreceivedbylabel`) now always return results + accounting for received coins from coinbase outputs, without an option to + change that behaviour. Excluding coinbases was previously deprecated in 23.0. + (#25171) + +- The `deprecatedrpc=fees` configuration option has been removed. The top-level + fee fields `fee`, `modifiedfee`, `ancestorfees` and `descendantfees` are no + longer returned by RPCs `getmempoolentry`, `getrawmempool(verbose=true)`, + `getmempoolancestors(verbose=true)` and `getmempooldescendants(verbose=true)`. + The same fee fields can be accessed through the `fees` object in the result. + The top-level fee fields were previously deprecated in 23.0. (#25204) + +- The `getpeerinfo` RPC has been updated with a new `presynced_headers` field, + indicating the progress on the presync phase mentioned in the + "P2P and network changes" section above. + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +- The `sendall` RPC spends specific UTXOs to one or more recipients + without creating change. By default, the `sendall` RPC will spend + every UTXO in the wallet. `sendall` is useful to empty wallets or to + create a changeless payment from select UTXOs. When creating a payment + from a specific amount for which the recipient incurs the transaction + fee, continue to use the `subtractfeefromamount` option via the + `send`, `sendtoaddress`, or `sendmany` RPCs. (#24118) + +- A new `gettxspendingprevout` RPC has been added, which scans the mempool to find + transactions spending any of the given outpoints. (#24408) + +- The `simulaterawtransaction` RPC iterates over the inputs and outputs of the given + transactions, and tallies up the balance change for the given wallet. This can be + useful e.g. when verifying that a coin join like transaction doesn't contain unexpected + inputs that the wallet will then sign for unintentionally. (#22751) + +Updated REST APIs +----------------- + +- The `/headers/` and `/blockfilterheaders/` endpoints have been updated to use + a query parameter instead of path parameter to specify the result count. The + count parameter is now optional, and defaults to 5 for both endpoints. The old + endpoints are still functional, and have no documented behaviour change. + + For `/headers`, use + `GET /rest/headers/.?count=` + instead of + `GET /rest/headers//.` (deprecated) + + For `/blockfilterheaders/`, use + `GET /rest/blockfilterheaders//.?count=` + instead of + `GET /rest/blockfilterheaders///.` (deprecated) + + (#24098) + +Build System +------------ + +- Guix builds are now reproducible across architectures (x86_64 & aarch64). (#21194) + +New settings +------------ + +- A new `mempoolfullrbf` option has been added, which enables the mempool to + accept transaction replacement without enforcing BIP125 replaceability + signaling. (#25353) + +Wallet +------ + +- The `-walletrbf` startup option will now default to `true`. The + wallet will now default to opt-in RBF on transactions that it creates. (#25610) + +- The `replaceable` option for the `createrawtransaction` and + `createpsbt` RPCs will now default to `true`. Transactions created + with these RPCs will default to having opt-in RBF enabled. (#25610) + +- The `wsh()` output descriptor was extended with Miniscript support. You can import Miniscript + descriptors for P2WSH in a watchonly wallet to track coins, but you can't spend from them using + the Bitcoin Core wallet yet. + You can find more about Miniscript on the [reference website](https://bitcoin.sipa.be/miniscript/). (#24148) + +- The `tr()` output descriptor now supports multisig scripts through the `multi_a()` and + `sortedmulti_a()` functions. (#24043) + +- To help prevent fingerprinting transactions created by the Bitcoin Core wallet, change output + amounts are now randomized. (#24494) + +- The `listtransactions`, `gettransaction`, and `listsinceblock` + RPC methods now include a wtxid field (hash of serialized transaction, + including witness data) for each transaction. (#24198) + +- The `listsinceblock`, `listtransactions` and `gettransaction` output now contain a new + `parent_descs` field for every "receive" entry. (#25504) + +- A new optional `include_change` parameter was added to the `listsinceblock` command. + +- RPC `getreceivedbylabel` now returns an error, "Label not found + in wallet" (-4), if the label is not in the address book. (#25122) + +Migrating Legacy Wallets to Descriptor Wallets +--------------------------------------------- + +An experimental RPC `migratewallet` has been added to migrate Legacy (non-descriptor) wallets to +Descriptor wallets. More information about the migration process is available in the +[documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/managing-wallets.md#migrating-legacy-wallets-to-descriptor-wallets). + +GUI changes +----------- + +- A new menu item to restore a wallet from a backup file has been added (gui#471). + +- Configuration changes made in the bitcoin GUI (such as the pruning setting, +proxy settings, UPNP preferences) are now saved to `/settings.json` +file rather than to the Qt settings backend (windows registry or unix desktop +config files), so these settings will now apply to bitcoind, instead of being +ignored. (#15936, gui#602) + +- Also, the interaction between GUI settings and `bitcoin.conf` settings is +simplified. Settings from `bitcoin.conf` are now displayed normally in the GUI +settings dialog, instead of in a separate warning message ("Options set in this +dialog are overridden by the configuration file: -setting=value"). And these +settings can now be edited because `settings.json` values take precedence over +`bitcoin.conf` values. (#15936) + +Low-level changes +================= + +RPC +--- + +- The `deriveaddresses`, `getdescriptorinfo`, `importdescriptors` and `scantxoutset` commands now + accept Miniscript expression within a `wsh()` descriptor. (#24148) + +- The `getaddressinfo`, `decodescript`, `listdescriptors` and `listunspent` commands may now output + a Miniscript descriptor inside a `wsh()` where a `wsh(raw())` descriptor was previously returned. (#24148) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- /dev/fd0 +- 0xb10c +- Adam Jonas +- akankshakashyap +- Ali Sherief +- amadeuszpawlik +- Andreas Kouloumos +- Andrew Chow +- Anthony Towns +- Antoine Poinsot +- Antoine Riard +- Aurèle Oulès +- avirgovi +- Ayush Sharma +- Baas +- Ben Woosley +- BrokenProgrammer +- brunoerg +- brydinh +- Bushstar +- Calvin Kim +- CAnon +- Carl Dong +- chinggg +- Cory Fields +- Daniel Kraft +- Daniela Brozzoni +- darosior +- Dave Scotese +- David Bakin +- dergoegge +- dhruv +- Dimitri +- dontbyte +- Duncan Dean +- eugene +- Eunoia +- Fabian Jahr +- furszy +- Gleb Naumenko +- glozow +- Greg Weber +- Gregory Sanders +- gruve-p +- Hennadii Stepanov +- hiago +- Igor Bubelov +- ishaanam +- Jacob P. +- Jadi +- James O'Beirne +- Janna +- Jarol Rodriguez +- Jeremy Rand +- Jeremy Rubin +- jessebarton +- João Barbosa +- John Newbery +- Jon Atack +- Josiah Baker +- Karl-Johan Alm +- KevinMusgrave +- Kiminuo +- klementtan +- Kolby Moroz +- kouloumos +- Kristaps Kaupe +- Larry Ruane +- Luke Dashjr +- MarcoFalke +- Marnix +- Martin Leitner-Ankerl +- Martin Zumsande +- Michael Dietz +- Michael Folkson +- Michael Ford +- Murch +- mutatrum +- muxator +- Oskar Mendel +- Pablo Greco +- pasta +- Patrick Strateman +- Pavol Rusnak +- Peter Bushnell +- phyBrackets +- Pieter Wuille +- practicalswift +- randymcmillan +- Robert Spigler +- Russell Yanofsky +- S3RK +- Samer Afach +- Sebastian Falbesoner +- Seibart Nedor +- Shashwat +- Sjors Provoost +- Smlep +- sogoagain +- Stacie +- Stéphan Vuylsteke +- Suhail Saqan +- Suhas Daftuar +- t-bast +- TakeshiMusgrave +- Vasil Dimov +- W. J. van der Laan +- w0xlt +- whiteh0rse +- willcl-ark +- William Casarin +- Yancy Ribbens + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-24.0.md b/doc/release-notes/release-notes-24.0.md new file mode 100644 index 000000000000..a0227aa17fc2 --- /dev/null +++ b/doc/release-notes/release-notes-24.0.md @@ -0,0 +1,4 @@ +Due to last-minute issues (#26616), 24.0, although tagged, was never fully +announced or released. + +See the release notes for 24.0.1 instead. diff --git a/doc/release-process.md b/doc/release-process.md index 0ac67b914647..17a03f7dcdb0 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -6,21 +6,22 @@ Release Process ### Before every release candidate * Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). -* Update manpages, see [gen-manpages.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagessh). * Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). +* Update manpages (after rebuilding the binaries), see [gen-manpages.py](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagespy). +* Update bitcoin.conf and commit, see [gen-bitcoin-conf.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-bitcoin-confsh). ### Before every major and minor release * Update [bips.md](bips.md) to account for changes since the last release (don't forget to bump the version number on the first line). * Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_RC` to `0`). +* Update manpages (see previous section) * Write release notes (see "Write the release notes" below). ### Before every major release * On both the master branch and the new release branch: - update `CLIENT_VERSION_MAJOR` in [`configure.ac`](../configure.ac) - - update `CLIENT_VERSION_MAJOR`, `PACKAGE_VERSION`, and `PACKAGE_STRING` in [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) -* On the new release branch in [`configure.ac`](../configure.ac) and [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) (see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd)): +* On the new release branch in [`configure.ac`](../configure.ac)(see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd)): - set `CLIENT_VERSION_MINOR` to `0` - set `CLIENT_VERSION_BUILD` to `0` - set `CLIENT_VERSION_IS_RELEASE` to `true` @@ -28,16 +29,23 @@ Release Process #### Before branch-off * Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/7415) for an example. -* Update [`src/chainparams.cpp`](/src/chainparams.cpp) m_assumed_blockchain_size and m_assumed_chain_state_size with the current size plus some overhead (see [this](#how-to-calculate-assumed-blockchain-and-chain-state-size) for information on how to calculate them). -* Update [`src/chainparams.cpp`](/src/chainparams.cpp) chainTxData with statistics about the transaction count and rate. Use the output of the `getchaintxstats` RPC, see - [this pull request](https://github.com/bitcoin/bitcoin/pull/20263) for an example. Reviewers can verify the results by running `getchaintxstats ` with the `window_block_count` and `window_final_block_hash` from your output. -* Update `src/chainparams.cpp` nMinimumChainWork and defaultAssumeValid (and the block height comment) with information from the `getblockheader` (and `getblockhash`) RPCs. - - The selected value must not be orphaned so it may be useful to set the value two blocks back from the tip. - - Testnet should be set some tens of thousands back from the tip due to reorgs there. - - This update should be reviewed with a reindex-chainstate with assumevalid=0 to catch any defect - that causes rejection of blocks in the past history. +* Update the following variables in [`src/chainparams.cpp`](/src/chainparams.cpp) for mainnet, testnet, and signet: + - `m_assumed_blockchain_size` and `m_assumed_chain_state_size` with the current size plus some overhead (see + [this](#how-to-calculate-assumed-blockchain-and-chain-state-size) for information on how to calculate them). + - The following updates should be reviewed with `reindex-chainstate` and `assumevalid=0` to catch any defect + that causes rejection of blocks in the past history. + - `chainTxData` with statistics about the transaction count and rate. Use the output of the `getchaintxstats` RPC with an + `nBlocks` of 4096 (28 days) and a `bestblockhash` of RPC `getbestblockhash`; see + [this pull request](https://github.com/bitcoin/bitcoin/pull/20263) for an example. Reviewers can verify the results by running + `getchaintxstats ` with the `window_block_count` and `window_final_block_hash` from your output. + - `defaultAssumeValid` with the output of RPC `getblockhash` using the `height` of `window_final_block_height` above + (and update the block height comment with that height), taking into account the following: + - On mainnet, the selected value must not be orphaned, so it may be useful to set the height two blocks back from the tip. + - Testnet should be set with a height some tens of thousands back from the tip, due to reorgs there. + - `nMinimumChainWork` with the "chainwork" value of RPC `getblockheader` using the same height as that selected for the previous step. - Clear the release notes and move them to the wiki (see "Write the release notes" below). -- Translations on Transifex +- Translations on Transifex: + - Pull translations from Transifex into the master branch. - Create [a new resource](https://www.transifex.com/bitcoin/bitcoin/content/) named after the major version with the slug `[bitcoin.qt-translation-x]`, where `RRR` is the major branch number padded with zeros. Use `src/qt/locale/bitcoin_en.xlf` to create it. - In the project workflow settings, ensure that [Translation Memory Fill-up](https://docs.transifex.com/translation-memory/enabling-autofill) is enabled and that [Translation Memory Context Matching](https://docs.transifex.com/translation-memory/translation-memory-with-context) is disabled. - Update the Transifex slug in [`.tx/config`](/.tx/config) to the slug of the resource created in the first step. This identifies which resource the translations will be synchronized from. @@ -47,20 +55,22 @@ Release Process #### After branch-off (on the major release branch) - Update the versions. +- Create the draft, named "*version* Release Notes Draft", as a [collaborative wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/_new). +- Clear the release notes: `cp doc/release-notes-empty-template.md doc/release-notes.md` - Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. - Translations on Transifex - Change the auto-update URL for the new major version's resource away from `master` and to the branch, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin//src/qt/locale/bitcoin_en.xlf`. Do not forget this or it will keep tracking the translations on master instead, drifting away from the specific major release. #### Before final release -- Merge the release notes from the wiki into the branch. +- Merge the release notes from [the wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/) into the branch. - Ensure the "Needs release note" label is removed from all relevant pull requests and issues. #### Tagging a release (candidate) To tag the version (or release candidate) in git, use the `make-tag.py` script from [bitcoin-maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools). From the root of the repository run: - ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 0.20.0) + ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 23.0) This will perform a few last-minute consistency checks in the build system files, and if they pass, create a signed tag. @@ -98,7 +108,7 @@ Checkout the Bitcoin Core version you'd like to build: pushd ./bitcoin SIGNER='(your builder key, ie bluematt, sipa, etc)' VERSION='(new version without v-prefix, e.g. 0.20.0)' -git fetch "v${VERSION}" +git fetch origin "v${VERSION}" git checkout "v${VERSION}" popd ``` @@ -110,57 +120,54 @@ against other `guix-attest` signatures. git -C ./guix.sigs pull ``` -### Create the macOS SDK tarball: (first time, or when SDK version changes) +### Create the macOS SDK tarball (first time, or when SDK version changes) Create the macOS SDK tarball, see the [macdeploy instructions](/contrib/macdeploy/README.md#deterministic-macos-dmg-notes) for details. -### Build and attest to build outputs: +### Build and attest to build outputs Follow the relevant Guix README.md sections: -- [Performing a build](/contrib/guix/README.md#performing-a-build) +- [Building](/contrib/guix/README.md#building) - [Attesting to build outputs](/contrib/guix/README.md#attesting-to-build-outputs) -### Verify other builders' signatures to your own. (Optional) - -Add other builders keys to your gpg keyring, and/or refresh keys: See `../bitcoin/contrib/builder-keys/README.md`. +### Verify other builders' signatures to your own (optional) -Follow the relevant Guix README.md sections: +- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md) - [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) -### Next steps: - -Commit your signature to guix.sigs: +### Commit your non codesigned signature to guix.sigs ```sh pushd ./guix.sigs git add "${VERSION}/${SIGNER}"/noncodesigned.SHA256SUMS{,.asc} -git commit -m "Add ${VERSION} unsigned sigs for ${SIGNER}" +git commit -m "Add attestations by ${SIGNER} for ${VERSION} non-codesigned" git push # Assuming you can push to the guix.sigs tree popd ``` -Codesigner only: Create Windows/macOS detached signatures: -- Only one person handles codesigning. Everyone else should skip to the next step. -- Only once the Windows/macOS builds each have 3 matching signatures may they be signed with their respective release keys. +## Codesigning -Codesigner only: Sign the macOS binary: +### macOS codesigner only: Create detached macOS signatures (assuming [signapple](https://github.com/achow101/signapple/) is installed and up to date with master branch) - transfer bitcoin-osx-unsigned.tar.gz to macOS for signing tar xf bitcoin-osx-unsigned.tar.gz - ./detached-sig-create.sh -s "Key ID" + ./detached-sig-create.sh /path/to/codesign.p12 Enter the keychain password and authorize the signature - Move signature-osx.tar.gz back to the guix-build host + signature-osx.tar.gz will be created -Codesigner only: Sign the windows binaries: +### Windows codesigner only: Create detached Windows signatures tar xf bitcoin-win-unsigned.tar.gz ./detached-sig-create.sh -key /path/to/codesign.key Enter the passphrase for the key when prompted signature-win.tar.gz will be created -Codesigner only: Commit the detached codesign payloads: +### Windows and macOS codesigners only: test code signatures +It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. +However if this is done, once the release has been tagged in the bitcoin-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds. + +### Windows and macOS codesigners only: Commit the detached codesign payloads ```sh pushd ./bitcoin-detached-sigs @@ -175,51 +182,48 @@ git push the current branch and new tag popd ``` -Non-codesigners: wait for Windows/macOS detached signatures: +### Non-codesigners: wait for Windows and macOS detached signatures -- Once the Windows/macOS builds each have 3 matching signatures, they will be signed with their respective release keys. +- Once the Windows and macOS builds each have 3 matching signatures, they will be signed with their respective release keys. - Detached signatures will then be committed to the [bitcoin-detached-sigs](https://github.com/bitcoin-core/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. -Create (and optionally verify) the codesigned outputs: +### Create the codesigned build outputs + +- [Codesigning build outputs](/contrib/guix/README.md#codesigning-build-outputs) -- [Codesigning](/contrib/guix/README.md#codesigning) +### Verify other builders' signatures to your own (optional) -Commit your signature for the signed macOS/Windows binaries: +- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md) +- [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) + +### Commit your codesigned signature to guix.sigs (for the signed macOS/Windows binaries) ```sh pushd ./guix.sigs git add "${VERSION}/${SIGNER}"/all.SHA256SUMS{,.asc} -git commit -m "Add ${SIGNER} ${VERSION} signed binaries signatures" +git commit -m "Add attestations by ${SIGNER} for ${VERSION} codesigned" git push # Assuming you can push to the guix.sigs tree popd ``` -### After 3 or more people have guix-built and their results match: - -Combine `all.SHA256SUMS` and `all.SHA256SUMS.asc` into a clear-signed -`SHA256SUMS.asc` message: - -```sh -echo -e "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\n$(cat all.SHA256SUMS)\n$(cat filename.txt.asc)" > SHA256SUMS.asc -``` +## After 3 or more people have guix-built and their results match -Here's an equivalent, more readable command if you're confident that you won't -mess up whitespaces when copy-pasting: +Combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`: ```bash -cat << EOF > SHA256SUMS.asc ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA256 - -$(cat all.SHA256SUMS) -$(cat all.SHA256SUMS.asc) -EOF +cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc ``` -- Upload to the bitcoincore.org server (`/var/www/bin/bitcoin-core-${VERSION}`): - 1. The contents of `./bitcoin/guix-build-${VERSION}/output`, except for + +- Upload to the bitcoincore.org server (`/var/www/bin/bitcoin-core-${VERSION}/`): + 1. The contents of each `./bitcoin/guix-build-${VERSION}/output/${HOST}/` directory, except for `*-debug*` files. + Guix will output all of the results into host subdirectories, but the SHA256SUMS + file does not include these subdirectories. In order for downloads via torrent + to verify without directory structure modification, all of the uploaded files + need to be in the same directory as the SHA256SUMS file. + The `*-debug*` files generated by the guix build contain debug symbols for troubleshooting by developers. It is assumed that anyone that is interested in debugging can run guix to generate the files for @@ -227,7 +231,13 @@ EOF as save storage space *do not upload these to the bitcoincore.org server, nor put them in the torrent*. - 2. The combined clear-signed message you just created `SHA256SUMS.asc` + ```sh + find guix-build-${VERSION}/output/ -maxdepth 2 -type f -not -name "SHA256SUMS.part" -and -not -name "*debug*" -exec scp {} user@bitcoincore.org:/var/www/bin/bitcoin-core-${VERSION} \; + ``` + + 2. The `SHA256SUMS` file + + 3. The `SHA256SUMS.asc` combined signature file you just created - Create a torrent of the `/var/www/bin/bitcoin-core-${VERSION}` directory such that at the top level there is only one file: the `bitcoin-core-${VERSION}` @@ -252,6 +262,10 @@ EOF - bitcoincore.org maintained versions update: [table](https://github.com/bitcoin-core/bitcoincore.org/commits/master/_includes/posts/maintenance-table.md) + - Delete post-EOL [release branches](https://github.com/bitcoin/bitcoin/branches/all) and create a tag `v${branch_name}-final`. + + - Delete ["Needs backport" labels](https://github.com/bitcoin/bitcoin/labels?q=backport) for non-existing branches. + - bitcoincore.org RPC documentation update - Install [golang](https://golang.org/doc/install) @@ -270,26 +284,7 @@ EOF - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 - - Push the latest version to master (if applicable), e.g. https://github.com/bitcoin-core/packaging/pull/32 - - - Create a new branch for the major release "0.xx" from master (used to build the snap package) and request the - track (if applicable), e.g. https://forum.snapcraft.io/t/track-request-for-bitcoin-core-snap/10112/7 - - - Notify MarcoFalke so that he can start building the snap package - - - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging (Click "Import Now" to fetch the branch) - - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging/+ref/0.xx (Click "Create snap package") - - Name it "bitcoin-core-snap-0.xx" - - Leave owner and series as-is - - Select architectures that are compiled via guix - - Leave "automatically build when branch changes" unticked - - Tick "automatically upload to store" - - Put "bitcoin-core" in the registered store package name field - - Tick the "edge" box - - Put "0.xx" in the track field - - Click "create snap package" - - Click "Request builds" for every new release on this branch (after updating the snapcraft.yml in the branch to reflect the latest guix results) - - Promote release on https://snapcraft.io/bitcoin-core/releases if it passes sanity checks + - Push the snap, see https://github.com/bitcoin-core/packaging/blob/master/snap/build.md - This repo @@ -314,15 +309,16 @@ EOF Both variables are used as a guideline for how much space the user needs on their drive in total, not just strictly for the blockchain. Note that all values should be taken from a **fully synced** node and have an overhead of 5-10% added on top of its base value. -To calculate `m_assumed_blockchain_size`: -- For `mainnet` -> Take the size of the data directory, excluding `/regtest` and `/testnet3` directories. -- For `testnet` -> Take the size of the `/testnet3` directory. - +To calculate `m_assumed_blockchain_size`, take the size in GiB of these directories: +- For `mainnet` -> the data directory, excluding the `/testnet3`, `/signet`, and `/regtest` directories and any overly large files, e.g. a huge `debug.log` +- For `testnet` -> `/testnet3` +- For `signet` -> `/signet` -To calculate `m_assumed_chain_state_size`: -- For `mainnet` -> Take the size of the `/chainstate` directory. -- For `testnet` -> Take the size of the `/testnet3/chainstate` directory. +To calculate `m_assumed_chain_state_size`, take the size in GiB of these directories: +- For `mainnet` -> `/chainstate` +- For `testnet` -> `/testnet3/chainstate` +- For `signet` -> `/signet/chainstate` Notes: - When taking the size for `m_assumed_blockchain_size`, there's no need to exclude the `/chainstate` directory since it's a guideline value and an overhead will be added anyway. -- The expected overhead for growth may change over time, so it may not be the same value as last release; pay attention to that when changing the variables. +- The expected overhead for growth may change over time. Consider whether the percentage needs to be changed in response; if so, update it here in this section. diff --git a/doc/tor.md b/doc/tor.md index 7d134b64e0e2..08d031d08443 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -16,17 +16,19 @@ configure Tor. ## How to see information about your Tor configuration via Bitcoin Core There are several ways to see your local onion address in Bitcoin Core: -- in the debug log (grep for "tor:" or "AddLocal") -- in the output of RPC `getnetworkinfo` in the "localaddresses" section -- in the output of the CLI `-netinfo` peer connections dashboard +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` +- in the debug log (grep for "AddLocal"; the Tor address ends in `.onion`) You may set the `-debug=tor` config logging option to have additional information in the debug log about your Tor configuration. -CLI `-addrinfo` returns the number of addresses known to your node per network -type, including Tor v2 and v3. This is useful to see how many onion addresses -are known to your node for `-onlynet=onion` and how many Tor v3 addresses it -knows when upgrading to Bitcoin Core v22.0 and up that supports Tor v3 only. +CLI `-addrinfo` returns the number of addresses known to your node per +network. This can be useful to see how many onion peers your node knows, +e.g. for `-onlynet=onion`. + +To fetch a number of onion addresses that your node knows, for example seven +addresses, use the `getnodeaddresses 7 onion` RPC. ## 1. Run Bitcoin Core behind a Tor proxy @@ -41,9 +43,11 @@ outgoing connections, but more is possible. -onion=ip:port Set the proxy server to use for Tor onion services. You do not need to set this if it's the same as -proxy. You can use -onion=0 to explicitly disable access to onion services. + ------------------------------------------------------------------ Note: Only the -proxy option sets the proxy for DNS requests; with -onion they will not route over Tor, so use -proxy if you have privacy concerns. + ------------------------------------------------------------------ -listen When using -proxy, listening is disabled by default. If you want to manually configure an onion service (see section 3), you'll @@ -54,14 +58,10 @@ outgoing connections, but more is possible. -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with other P2P nodes. - -onlynet=onion Make outgoing connections only to .onion addresses. Incoming - connections are not affected by this option. This option can be - specified multiple times to allow multiple network types, e.g. - ipv4, ipv6 or onion. If you use this option with values other - than onion you *cannot* disable onion connections; outgoing onion - connections will be enabled when you use -proxy or -onion. Use - -noonion or -onion=0 if you want to be sure there are no outbound - onion connections over the default proxy or your defined -proxy. + -onlynet=onion Make automatic outbound connections only to .onion addresses. + Inbound and manual connections are not affected by this option. + It can be specified multiple times to allow multiple networks, + e.g. onlynet=onion, onlynet=i2p, onlynet=cjdns. In a typical situation, this suffices to run behind a Tor proxy: @@ -134,7 +134,7 @@ You can also check the group of the cookie file. On most Linux systems, the Tor auth cookie will usually be `/run/tor/control.authcookie`: ``` -stat -c '%G' /run/tor/control.authcookie +TORGROUP=$(stat -c '%G' /run/tor/control.authcookie) ``` Once you have determined the `${TORGROUP}` and selected the `${USER}` that will diff --git a/doc/tracing.md b/doc/tracing.md new file mode 100644 index 000000000000..6e60901496b2 --- /dev/null +++ b/doc/tracing.md @@ -0,0 +1,362 @@ +# User-space, Statically Defined Tracing (USDT) for Bitcoin Core + +Bitcoin Core includes statically defined tracepoints to allow for more +observability during development, debugging, code review, and production usage. +These tracepoints make it possible to keep track of custom statistics and +enable detailed monitoring of otherwise hidden internals. They have +little to no performance impact when unused. + +``` +eBPF and USDT Overview +====================== + + ┌──────────────────┐ ┌──────────────┐ + │ tracing script │ │ bitcoind │ + │==================│ 2. │==============│ + │ eBPF │ tracing │ hooks │ │ + │ code │ logic │ into┌─┤►tracepoint 1─┼───┐ 3. + └────┬───┴──▲──────┘ ├─┤►tracepoint 2 │ │ pass args + 1. │ │ 4. │ │ ... │ │ to eBPF + User compiles │ │ pass data to │ └──────────────┘ │ program + Space & loads │ │ tracing script │ │ + ─────────────────┼──────┼─────────────────┼────────────────────┼─── + Kernel │ │ │ │ + Space ┌──┬─▼──────┴─────────────────┴────────────┐ │ + │ │ eBPF program │◄──────┘ + │ └───────────────────────────────────────┤ + │ eBPF kernel Virtual Machine (sandboxed) │ + └──────────────────────────────────────────┘ + +1. The tracing script compiles the eBPF code and loads the eBPF program into a kernel VM +2. The eBPF program hooks into one or more tracepoints +3. When the tracepoint is called, the arguments are passed to the eBPF program +4. The eBPF program processes the arguments and returns data to the tracing script +``` + +The Linux kernel can hook into the tracepoints during runtime and pass data to +sandboxed [eBPF] programs running in the kernel. These eBPF programs can, for +example, collect statistics or pass data back to user-space scripts for further +processing. + +[eBPF]: https://ebpf.io/ + +The two main eBPF front-ends with support for USDT are [bpftrace] and +[BPF Compiler Collection (BCC)]. BCC is used for complex tools and daemons and +`bpftrace` is preferred for one-liners and shorter scripts. Examples for both can +be found in [contrib/tracing]. + +[bpftrace]: https://github.com/iovisor/bpftrace +[BPF Compiler Collection (BCC)]: https://github.com/iovisor/bcc +[contrib/tracing]: ../contrib/tracing/ + +## Tracepoint documentation + +The currently available tracepoints are listed here. + +### Context `net` + +#### Tracepoint `net:inbound_message` + +Is called when a message is received from a peer over the P2P network. Passes +information about our peer, the connection and the message as arguments. + +Arguments passed: +1. Peer ID as `int64` +2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (max. length 68 characters) +3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters) +4. Message Type (inv, ping, getdata, addrv2, ...) as `pointer to C-style String` (max. length 20 characters) +5. Message Size in bytes as `uint64` +6. Message Bytes as `pointer to unsigned chars` (i.e. bytes) + +Note: The message is passed to the tracepoint in full, however, due to space +limitations in the eBPF kernel VM it might not be possible to pass the message +to user-space in full. Messages longer than a 32kb might be cut off. This can +be detected in tracing scripts by comparing the message size to the length of +the passed message. + +#### Tracepoint `net:outbound_message` + +Is called when a message is sent to a peer over the P2P network. Passes +information about our peer, the connection and the message as arguments. + +Arguments passed: +1. Peer ID as `int64` +2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (max. length 68 characters) +3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters) +4. Message Type (inv, ping, getdata, addrv2, ...) as `pointer to C-style String` (max. length 20 characters) +5. Message Size in bytes as `uint64` +6. Message Bytes as `pointer to unsigned chars` (i.e. bytes) + +Note: The message is passed to the tracepoint in full, however, due to space +limitations in the eBPF kernel VM it might not be possible to pass the message +to user-space in full. Messages longer than a 32kb might be cut off. This can +be detected in tracing scripts by comparing the message size to the length of +the passed message. + +### Context `validation` + +#### Tracepoint `validation:block_connected` + +Is called *after* a block is connected to the chain. Can, for example, be used +to benchmark block connections together with `-reindex`. + +Arguments passed: +1. Block Header Hash as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Block Height as `int32` +3. Transactions in the Block as `uint64` +4. Inputs spend in the Block as `int32` +5. SigOps in the Block (excluding coinbase SigOps) `uint64` +6. Time it took to connect the Block in microseconds (µs) as `uint64` + +### Context `utxocache` + +The following tracepoints cover the in-memory UTXO cache. UTXOs are, for example, +added to and removed (spent) from the cache when we connect a new block. +**Note**: Bitcoin Core uses temporary clones of the _main_ UTXO cache +(`chainstate.CoinsTip()`). For example, the RPCs `generateblock` and +`getblocktemplate` call `TestBlockValidity()`, which applies the UTXO set +changes to a temporary cache. Similarly, mempool consistency checks, which are +frequent on regtest, also apply the UTXO set changes to a temporary cache. +Changes to the _main_ UTXO cache and to temporary caches trigger the tracepoints. +We can't tell if a temporary cache or the _main_ cache was changed. + +#### Tracepoint `utxocache:flush` + +Is called *after* the in-memory UTXO cache is flushed. + +Arguments passed: +1. Time it took to flush the cache microseconds as `int64` +2. Flush state mode as `uint32`. It's an enumerator class with values `0` + (`NONE`), `1` (`IF_NEEDED`), `2` (`PERIODIC`), `3` (`ALWAYS`) +3. Cache size (number of coins) before the flush as `uint64` +4. Cache memory usage in bytes as `uint64` +5. If pruning caused the flush as `bool` + +#### Tracepoint `utxocache:add` + +Is called when a coin is added to a UTXO cache. This can be a temporary UTXO cache too. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was added to the UTXO-set as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +#### Tracepoint `utxocache:spent` + +Is called when a coin is spent from a UTXO cache. This can be a temporary UTXO cache too. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was spent, as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +#### Tracepoint `utxocache:uncache` + +Is called when a coin is purposefully unloaded from a UTXO cache. This +happens, for example, when we load an UTXO into a cache when trying to accept +a transaction that turns out to be invalid. The loaded UTXO is uncached to avoid +filling our UTXO cache up with irrelevant UTXOs. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was uncached, as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +### Context `coin_selection` + +#### Tracepoint `coin_selection:selected_coins` + +Is called when `SelectCoins` completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Coin selection algorithm name as `pointer to C-style string` +3. Selection target value as `int64` +4. Calculated waste metric of the solution as `int64` +5. Total value of the selected inputs as `int64` + +#### Tracepoint `coin_selection:normal_create_tx_internal` + +Is called when the first `CreateTransactionInternal` completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Whether `CreateTransactionInternal` succeeded as `bool` +3. The expected transaction fee as an `int64` +4. The position of the change output as an `int32` + +#### Tracepoint `coin_selection:attempting_aps_create_tx` + +Is called when `CreateTransactionInternal` is called the second time for the optimistic +Avoid Partial Spends selection attempt. This is used to determine whether the next +tracepoints called are for the Avoid Partial Spends solution, or a different transaction. + +Arguments passed: +1. Wallet name as `pointer to C-style string` + +#### Tracepoint `coin_selection:aps_create_tx_internal` + +Is called when the second `CreateTransactionInternal` with Avoid Partial Spends enabled completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Whether the Avoid Partial Spends solution will be used as `bool` +3. Whether `CreateTransactionInternal` succeeded as` bool` +4. The expected transaction fee as an `int64` +5. The position of the change output as an `int32` + +## Adding tracepoints to Bitcoin Core + +To add a new tracepoint, `#include ` in the compilation unit where +the tracepoint is inserted. Use one of the `TRACEx` macros listed below +depending on the number of arguments passed to the tracepoint. Up to 12 +arguments can be provided. The `context` and `event` specify the names by which +the tracepoint is referred to. Please use `snake_case` and try to make sure that +the tracepoint names make sense even without detailed knowledge of the +implementation details. Do not forget to update the tracepoint list in this +document. + +```c +#define TRACE(context, event) +#define TRACE1(context, event, a) +#define TRACE2(context, event, a, b) +#define TRACE3(context, event, a, b, c) +#define TRACE4(context, event, a, b, c, d) +#define TRACE5(context, event, a, b, c, d, e) +#define TRACE6(context, event, a, b, c, d, e, f) +#define TRACE7(context, event, a, b, c, d, e, f, g) +#define TRACE8(context, event, a, b, c, d, e, f, g, h) +#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) +#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) +#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) +#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) +``` + +For example: + +```C++ +TRACE6(net, inbound_message, + pnode->GetId(), + pnode->m_addr_name.c_str(), + pnode->ConnectionTypeAsString().c_str(), + sanitizedType.c_str(), + msg.data.size(), + msg.data.data() +); +``` + +### Guidelines and best practices + +#### Clear motivation and use case +Tracepoints need a clear motivation and use case. The motivation should +outweigh the impact on, for example, code readability. There is no point in +adding tracepoints that don't end up being used. + +#### Provide an example +When adding a new tracepoint, provide an example. Examples can show the use case +and help reviewers testing that the tracepoint works as intended. The examples +can be kept simple but should give others a starting point when working with +the tracepoint. See existing examples in [contrib/tracing/]. + +[contrib/tracing/]: ../contrib/tracing/ + +#### No expensive computations for tracepoints +Data passed to the tracepoint should be inexpensive to compute. Although the +tracepoint itself only has overhead when enabled, the code to compute arguments +is always run - even if the tracepoint is not used. For example, avoid +serialization and parsing. + +#### Semi-stable API +Tracepoints should have a semi-stable API. Users should be able to rely on the +tracepoints for scripting. This means tracepoints need to be documented, and the +argument order ideally should not change. If there is an important reason to +change argument order, make sure to document the change and update the examples +using the tracepoint. + +#### eBPF Virtual Machine limits +Keep the eBPF Virtual Machine limits in mind. eBPF programs receiving data from +the tracepoints run in a sandboxed Linux kernel VM. This VM has a limited stack +size of 512 bytes. Check if it makes sense to pass larger amounts of data, for +example, with a tracing script that can handle the passed data. + +#### `bpftrace` argument limit +While tracepoints can have up to 12 arguments, bpftrace scripts currently only +support reading from the first six arguments (`arg0` till `arg5`) on `x86_64`. +bpftrace currently lacks real support for handling and printing binary data, +like block header hashes and txids. When a tracepoint passes more than six +arguments, then string and integer arguments should preferably be placed in the +first six argument fields. Binary data can be placed in later arguments. The BCC +supports reading from all 12 arguments. + +#### Strings as C-style String +Generally, strings should be passed into the `TRACEx` macros as pointers to +C-style strings (a null-terminated sequence of characters). For C++ +`std::strings`, [`c_str()`] can be used. It's recommended to document the +maximum expected string size if known. + + +[`c_str()`]: https://www.cplusplus.com/reference/string/string/c_str/ + + +## Listing available tracepoints + +Multiple tools can list the available tracepoints in a `bitcoind` binary with +USDT support. + +### GDB - GNU Project Debugger + +To list probes in Bitcoin Core, use `info probes` in `gdb`: + +``` +$ gdb ./src/bitcoind +… +(gdb) info probes +Type Provider Name Where Semaphore Object +stap net inbound_message 0x000000000014419e /src/bitcoind +stap net outbound_message 0x0000000000107c05 /src/bitcoind +stap validation block_connected 0x00000000002fb10c /src/bitcoind +… +``` + +### With `readelf` + +The `readelf` tool can be used to display the USDT tracepoints in Bitcoin Core. +Look for the notes with the description `NT_STAPSDT`. + +``` +$ readelf -n ./src/bitcoind | grep NT_STAPSDT -A 4 -B 2 +Displaying notes found in: .note.stapsdt + Owner Data size Description + stapsdt 0x0000005d NT_STAPSDT (SystemTap probe descriptors) + Provider: net + Name: outbound_message + Location: 0x0000000000107c05, Base: 0x0000000000579c90, Semaphore: 0x0000000000000000 + Arguments: -8@%r12 8@%rbx 8@%rdi 8@192(%rsp) 8@%rax 8@%rdx +… +``` + +### With `tplist` + +The `tplist` tool is provided by BCC (see [Installing BCC]). It displays kernel +tracepoints or USDT probes and their formats (for more information, see the +[`tplist` usage demonstration]). There are slight binary naming differences +between distributions. For example, on +[Ubuntu the binary is called `tplist-bpfcc`][ubuntu binary]. + +[Installing BCC]: https://github.com/iovisor/bcc/blob/master/INSTALL.md +[`tplist` usage demonstration]: https://github.com/iovisor/bcc/blob/master/tools/tplist_example.txt +[ubuntu binary]: https://github.com/iovisor/bcc/blob/master/INSTALL.md#ubuntu---binary + +``` +$ tplist -l ./src/bitcoind -v +b'net':b'outbound_message' [sema 0x0] + 1 location(s) + 6 argument(s) +… +``` diff --git a/doc/zmq.md b/doc/zmq.md index 85f33701306b..4055505d7480 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -76,7 +76,7 @@ The option to set the PUB socket's outbound message high water mark -zmqpubhashblockhwm=n -zmqpubrawblockhwm=n -zmqpubrawtxhwm=n - -zmqpubsequencehwm=address + -zmqpubsequencehwm=n The high water mark value must be an integer greater than or equal to 0. @@ -84,15 +84,18 @@ For instance: $ bitcoind -zmqpubhashtx=tcp://127.0.0.1:28332 \ -zmqpubhashtx=tcp://192.168.1.2:28332 \ + -zmqpubhashblock="tcp://[::1]:28333" \ -zmqpubrawtx=ipc:///tmp/bitcoind.tx.raw \ -zmqpubhashtxhwm=10000 Each PUB notification has a topic and body, where the header corresponds to the notification type. For instance, for the notification `-zmqpubhashtx` the topic is `hashtx` (no null -terminator) and the body is the transaction hash (32 -bytes) for all but `sequence` topic. For `sequence`, the body -is structured as the following based on the type of message: +terminator). These options can also be provided in bitcoin.conf. + +The topics are: + +`sequence`: the body is structured as the following based on the type of message: <32-byte hash>C : Blockhash connected <32-byte hash>D : Blockhash disconnected @@ -101,7 +104,24 @@ is structured as the following based on the type of message: Where the 8-byte uints correspond to the mempool sequence number. -These options can also be provided in bitcoin.conf. +`rawtx`: Notifies about all transactions, both when they are added to mempool or when a new block arrives. This means a transaction could be published multiple times. First, when it enters the mempool and then again in each block that includes it. The messages are ZMQ multipart messages with three parts. The first part is the topic (`rawtx`), the second part is the serialized transaction, and the last part is a sequence number (representing the message count to detect lost messages). + + | rawtx | | + +`hashtx`: Notifies about all transactions, both when they are added to mempool or when a new block arrives. This means a transaction could be published multiple times. First, when it enters the mempool and then again in each block that includes it. The messages are ZMQ multipart messages with three parts. The first part is the topic (`hashtx`), the second part is the 32-byte transaction hash, and the last part is a sequence number (representing the message count to detect lost messages). + + | hashtx | <32-byte transaction hash in Little Endian> | + + +`rawblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`rawblock`), the second part is the serialized block, and the last part is a sequence number (representing the message count to detect lost messages). + + | rawblock | | + +`hashblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`hashblock`), the second part is the 32-byte block hash, and the last part is a sequence number (representing the message count to detect lost messages). + + | hashblock | <32-byte block hash in Little Endian> | + +**_NOTE:_** Note that the 32-byte hashes are in Little Endian and not in the Big Endian format that the RPC interface and block explorers use to display transaction and block hashes. ZeroMQ endpoint specifiers for TCP (and others) are documented in the [ZeroMQ API](http://api.zeromq.org/4-0:_start). @@ -125,6 +145,9 @@ Setting the keepalive values appropriately for your operating environment may improve connectivity in situations where long-lived connections are silently dropped by network middle boxes. +Also, the socket's ZMQ_IPV6 option is enabled to accept connections from IPv6 +hosts as well. If needed, this option has to be set on the client side too. + ## Remarks From the perspective of bitcoind, the ZeroMQ socket is write-only; PUB diff --git a/hex.txt b/hex.txt new file mode 100644 index 000000000000..acf17b02d1a6 --- /dev/null +++ b/hex.txt @@ -0,0 +1 @@ +010000000001015c946d56d9e0bf295a72a4b1b3c030382e83e98b1d60da4c0ecca45946f2ca2e0000000000ffffffff1f62dc94ad000000001600148453d78e0d7fc1f62ae60e2e376732a98072c509c79b090000000000160014b2d2999647cca60e5c6d4d867a67d27c5897ab7880380100000000001600146f04999bc9486511113f874a05592128df803c0ed730540000000000160014a4bead8f41bac4b853e26a686275b69d7ac69246f6a22600000000001600149d4acbae9f6ffc747123a182b6edb6b4fb253bdf94a91600000000001976a914d247c6edd4a01949cf16c168c6743d68b224238388ac0a91471f000000001600145897a3bfdf54fffa6f342c0267495e25565a8475b2b2140000000000160014096bb1740c20cce2cf3e18f20ee3964e6f3a22712c40920000000000160014db1c5404ee46cc14646bd90d693eae8e82da333f60011200000000001600142b327f3f533f3c7f28339c3e547fb361de2621d1eb930c000000000016001429e4fb4675f8ae688cb805d9afc05d01bc988b862ae1480000000000160014de71bb5b598131137a49402ceda92f4bfd08fbb3e073eb0b0000000017a914d53ba3ddeb90a6a86eb73d0d334d36f608ecf62d87723b09000000000017a914a10bac8a3f4eaaf34d2967501ed70394573ea5a68735952f00000000001600140bd4f36828bcfade301709628d288218aa9010ccc0b704000000000017a9145794ca4738bb803368f42a4d5e7fce246d497253878d7904000000000017a914cf1000740db0a252f16276b445e897d0196ed18287a0f8e8000000000016001493775437ef77d8a9aac88a545e1fc650a34c34630f79c90100000000160014b5a17b64b6d649abd8028ba0b84704d1616327b48fd783020000000017a9145d6d497e0665cba2e7d7fa6c6752f1d31a8f6fe387828602000000000017a91446af9c627be0107304a614a6c4da6cd5638c1e5b8768c0f500000000001976a914cf58dc13bd71843d2f5a79916a3848e84b34bcd188ac68f40c000000000017a91428a01c480c78c12a54acb06b59d14f66e0d6231487503f16000000000017a9148f4adebc88d6e7650f30f208fd1af82e2e7e178c8778195f000000000017a914704284ef35a1d0ff15de64ecb5e345d4fe673e3587006cdc020000000017a9147d48d036a51a11055d3af4ce3e6e5985e201636a87005307000000000017a9143eb297e0afe56be949d5efe30d9cc8af1f2365118730fd1300000000001600148b5b65a49c677e5ae01d0bb233d1efc8eb14e99d74df2500000000001600147f2280844933a0dc045dea525f6e78575a9d78a550971201000000001600144ce0f42097168771e8474bb4e9144088501250a7d566fc6f01000000160014dc6bf86354105de2fcd9868a2b0376d6731cb92f024730440220078d40012133b9c9d3c7c27174c74328d4fa055d3451bfd70ebe3a8b5772927302207b3c9025ab12316f37e0d8f744ac5b2bad954387cd51898c2b4e5e8ca13eeff1012102174ee672429ff94304321cdae1fc1e487edf658b34bd1d36da03761658a2bb0900000000 \ No newline at end of file diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 4a947001fa2d..5bee4bf92e73 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -1,191 +1 @@ -## -## bitcoin.conf configuration file. Lines beginning with # are comments. -## - -# Network-related settings: - -# Note that if you use testnet, signet or regtest, particularly with the options -# addnode, connect, port, bind, rpcport, rpcbind or wallet, you will also -# want to read "[Sections]" further down. - -# Run on the testnet network -#testnet=0 - -# Run on a signet network -#signet=0 - -# Run a regression test network -#regtest=0 - -# Connect via a SOCKS5 proxy -#proxy=127.0.0.1:9050 - -# Bind to given address and always listen on it. Use [host]:port notation for IPv6 -#bind= - -# Bind to given address and add permission flags to peers connecting to it. Use [host]:port notation for IPv6 -#whitebind=perm@ - -############################################################## -## Quick Primer on addnode vs connect ## -## Let's say for instance you use addnode=4.2.2.4 ## -## addnode will connect you to and tell you about the ## -## nodes connected to 4.2.2.4. In addition it will tell ## -## the other nodes connected to it that you exist so ## -## they can connect to you. ## -## connect will not do the above when you 'connect' to it. ## -## It will *only* connect you to 4.2.2.4 and no one else.## -## ## -## So if you're behind a firewall, or have other problems ## -## finding nodes, add some using 'addnode'. ## -## ## -## If you want to stay private, use 'connect' to only ## -## connect to "trusted" nodes. ## -## ## -## If you run multiple nodes on a LAN, there's no need for ## -## all of them to open lots of connections. Instead ## -## 'connect' them all to one node that is port forwarded ## -## and has lots of connections. ## -## Thanks goes to [Noodle] on Freenode. ## -############################################################## - -# Use as many addnode= settings as you like to connect to specific peers -#addnode=69.164.218.197 -#addnode=10.0.0.2:8333 - -# Alternatively use as many connect= settings as you like to connect ONLY to specific peers -#connect=69.164.218.197 -#connect=10.0.0.1:8333 - -# Listening mode, enabled by default except when 'connect' is being used -#listen=1 - -# Port on which to listen for connections (default: 8333, testnet: 18333, signet: 38333, regtest: 18444) -#port= - -# Maximum number of inbound + outbound connections (default: 125). This option -# applies only if inbound connections are enabled; otherwise, the number of connections -# will not be more than 11: 8 full-relay connections, 2 block-relay-only ones, and -# occasionally 1 short-lived feeler or extra outbound block-relay-only connection. -# These limits do not apply to connections added manually with the -addnode -# configuration option or the addnode RPC, which have a separate limit of 8 connections. -#maxconnections= - -# Maximum upload bandwidth target in MiB per day (e.g. 'maxuploadtarget=1024' is 1 GiB per day). -# This limits the upload bandwidth for those with bandwidth limits. 0 = no limit (default: 0). -# -maxuploadtarget does not apply to peers with 'download' permission. -# For more information on reducing bandwidth utilization, see: doc/reduce-traffic.md. -#maxuploadtarget= - -# -# JSON-RPC options (for controlling a running Bitcoin/bitcoind process) -# - -# server=1 tells Bitcoin-Qt and bitcoind to accept JSON-RPC commands -#server=0 - -# Bind to given address to listen for JSON-RPC connections. -# Refer to the manpage or bitcoind -help for further details. -#rpcbind= - -# If no rpcpassword is set, rpc cookie auth is sought. The default `-rpccookiefile` name -# is .cookie and found in the `-datadir` being used for bitcoind. This option is typically used -# when the server and client are run as the same user. -# -# If not, you must set rpcuser and rpcpassword to secure the JSON-RPC API. -# -# The config option `rpcauth` can be added to server startup argument. It is set at initialization time -# using the output from the script in share/rpcauth/rpcauth.py after providing a username: -# -# ./share/rpcauth/rpcauth.py alice -# String to be appended to bitcoin.conf: -# rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae -# Your password: -# DONT_USE_THIS_YOU_WILL_GET_ROBBED_8ak1gI25KFTvjovL3gAM967mies3E= -# -# On client-side, you add the normal user/password pair to send commands: -#rpcuser=alice -#rpcpassword=DONT_USE_THIS_YOU_WILL_GET_ROBBED_8ak1gI25KFTvjovL3gAM967mies3E= -# -# You can even add multiple entries of these to the server conf file, and client can use any of them: -# rpcauth=bob:b2dd077cb54591a2f3139e69a897ac$4e71f08d48b4347cf8eff3815c0e25ae2e9a4340474079f55705f40574f4ec99 - -# How many seconds bitcoin will wait for a complete RPC HTTP request. -# after the HTTP connection is established. -#rpcclienttimeout=30 - -# By default, only RPC connections from localhost are allowed. -# Specify as many rpcallowip= settings as you like to allow connections from other hosts, -# either as a single IPv4/IPv6 or with a subnet specification. - -# NOTE: opening up the RPC port to hosts outside your local trusted network is NOT RECOMMENDED, -# because the rpcpassword is transmitted over the network unencrypted. - -# server=1 tells Bitcoin-Qt to accept JSON-RPC commands. -# it is also read by bitcoind to determine if RPC should be enabled -#rpcallowip=10.1.1.34/255.255.255.0 -#rpcallowip=1.2.3.4/24 -#rpcallowip=2001:db8:85a3:0:0:8a2e:370:7334/96 - -# Listen for RPC connections on this TCP port: -#rpcport=8332 - -# You can use Bitcoin or bitcoind to send commands to Bitcoin/bitcoind -# running on another host using this option: -#rpcconnect=127.0.0.1 - -# Wallet options - -# Specify where to find wallet, lockfile and logs. If not present, those files will be -# created as new. -#wallet=
- -# Create transactions that have enough fees so they are likely to begin confirmation within n blocks (default: 6). -# This setting is over-ridden by the -paytxfee option. -#txconfirmtarget=n - -# Pay a transaction fee every time you send bitcoins. -#paytxfee=0.000x - -# Miscellaneous options - -# Pre-generate this many public/private key pairs, so wallet backups will be valid for -# both prior transactions and several dozen future transactions. -#keypool=100 - -# Maintain coinstats index used by the gettxoutsetinfo RPC (default: 0). -#coinstatsindex=1 - -# Enable pruning to reduce storage requirements by deleting old blocks. -# This mode is incompatible with -txindex, -coinstatsindex and -rescan. -# 0 = default (no pruning). -# 1 = allows manual pruning via RPC. -# >=550 = target to stay under in MiB. -#prune=550 - -# User interface options - -# Start Bitcoin minimized -#min=1 - -# Minimize to the system tray -#minimizetotray=1 - -# [Sections] -# Most options apply to mainnet, testnet, signet and regtest. -# If you want to confine an option to just one network, you should add it in the -# relevant section below. -# EXCEPTIONS: The options addnode, connect, port, bind, rpcport, rpcbind and wallet -# only apply to mainnet unless they appear in the appropriate section below. - -# Options only for mainnet -[main] - -# Options only for testnet -[test] - -# Options only for signet -[signet] - -# Options only for regtest -[regtest] +# This is a placeholder file. Please follow the instructions in `contrib/devtools/README.md` to generate a bitcoin.conf file. diff --git a/share/genbuild.sh b/share/genbuild.sh index fe89ae1fdefc..ecc96160e690 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2012-2020 The Bitcoin Core developers +# Copyright (c) 2012-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -26,7 +26,7 @@ if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$ # if latest commit is tagged and not dirty, then override using the tag name RAWDESC=$(git describe --abbrev=0 2>/dev/null) - if [ "$(git rev-parse HEAD)" = "$(git rev-list -1 $RAWDESC 2>/dev/null)" ]; then + if [ "$(git rev-parse HEAD)" = "$(git rev-list -1 "$RAWDESC" 2>/dev/null)" ]; then git diff-index --quiet HEAD -- && GIT_TAG=$RAWDESC fi diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index da10dbb3be1b..053359e0a887 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ LSMinimumSystemVersion - 10.14.0 + 10.15.0 LSArchitecturePriority @@ -16,6 +16,11 @@ CFBundlePackageType APPL + CFBundleSupportedPlatforms + + MacOSX + + NSHumanReadableCopyright @CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_BUILD@, Copyright © 2009-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@ diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index e4d438fdb007..39acec89423b 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2019 The Bitcoin Core developers +# Copyright (c) 2012-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/share/rpcauth/rpcauth.py b/share/rpcauth/rpcauth.py index b14c80171e8f..d441d5f21d4c 100755 --- a/share/rpcauth/rpcauth.py +++ b/share/rpcauth/rpcauth.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. from argparse import ArgumentParser from base64 import urlsafe_b64encode -from binascii import hexlify from getpass import getpass from os import urandom @@ -13,7 +12,7 @@ def generate_salt(size): """Create size byte hex salt""" - return hexlify(urandom(size)).decode() + return urandom(size).hex() def generate_password(): """Create 32 byte b64 password""" diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 85ae7c57af26..2ce798bd2d1d 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -53,7 +53,7 @@ Var StartMenuGroup # Installer attributes InstallDir $PROGRAMFILES64\Bitcoin -CRCCheck on +CRCCheck force XPStyle on BrandingText " " ShowInstDetails show @@ -72,16 +72,18 @@ ShowUninstDetails show Section -Main SEC0000 SetOutPath $INSTDIR SetOverwrite on - File @abs_top_srcdir@/release/@BITCOIN_GUI_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_GUI_NAME@@EXEEXT@ File /oname=COPYING.txt @abs_top_srcdir@/COPYING File /oname=readme.txt @abs_top_srcdir@/doc/README_windows.txt + File @abs_top_srcdir@/share/examples/bitcoin.conf + SetOutPath $INSTDIR\share\rpcauth + File @abs_top_srcdir@/share/rpcauth/*.* SetOutPath $INSTDIR\daemon - File @abs_top_srcdir@/release/@BITCOIN_DAEMON_NAME@@EXEEXT@ - File @abs_top_srcdir@/release/@BITCOIN_CLI_NAME@@EXEEXT@ - File @abs_top_srcdir@/release/@BITCOIN_TX_NAME@@EXEEXT@ - File @abs_top_srcdir@/release/@BITCOIN_WALLET_TOOL_NAME@@EXEEXT@ - SetOutPath $INSTDIR\doc - File /r /x Makefile* @abs_top_srcdir@/doc\*.* + File @abs_top_builddir@/release/@BITCOIN_DAEMON_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_CLI_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_TX_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_WALLET_TOOL_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_TEST_NAME@@EXEEXT@ SetOutPath $INSTDIR WriteRegStr HKCU "${REGKEY}\Components" Main 1 SectionEnd @@ -128,8 +130,9 @@ Section /o -un.Main UNSEC0000 Delete /REBOOTOK $INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@ Delete /REBOOTOK $INSTDIR\COPYING.txt Delete /REBOOTOK $INSTDIR\readme.txt + Delete /REBOOTOK $INSTDIR\bitcoin.conf + RMDir /r /REBOOTOK $INSTDIR\share RMDir /r /REBOOTOK $INSTDIR\daemon - RMDir /r /REBOOTOK $INSTDIR\doc DeleteRegValue HKCU "${REGKEY}\Components" Main SectionEnd diff --git a/src/.bear-tidy-config b/src/.bear-tidy-config new file mode 100644 index 000000000000..111ef6ee444a --- /dev/null +++ b/src/.bear-tidy-config @@ -0,0 +1,15 @@ +{ + "output": { + "content": { + "include_only_existing_source": true, + "paths_to_include": [], + "paths_to_exclude": [ + "src/leveldb" + ] + }, + "format": { + "command_as_array": true, + "drop_output_field": false + } + } +} diff --git a/src/.clang-format b/src/.clang-format index a69c57f3e0de..791b3b8f9f06 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -34,14 +34,6 @@ IndentWidth: 4 KeepEmptyLinesAtTheStartOfBlocks: false MaxEmptyLinesToKeep: 2 NamespaceIndentation: None -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: false -PenaltyBreakBeforeFirstCallParameter: 1 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements @@ -51,6 +43,5 @@ SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false -Standard: Cpp11 -TabWidth: 8 +Standard: c++17 UseTab: Never diff --git a/src/.clang-tidy b/src/.clang-tidy new file mode 100644 index 000000000000..9d78ccc95965 --- /dev/null +++ b/src/.clang-tidy @@ -0,0 +1,28 @@ +Checks: ' +-*, +bugprone-argument-comment, +bugprone-use-after-move, +misc-unused-using-decls, +modernize-use-default-member-init, +modernize-use-nullptr, +performance-for-range-copy, +performance-move-const-arg, +performance-unnecessary-copy-initialization, +readability-redundant-declaration, +readability-redundant-string-init, +' +WarningsAsErrors: ' +bugprone-argument-comment, +bugprone-use-after-move, +misc-unused-using-decls, +modernize-use-default-member-init, +modernize-use-nullptr, +performance-for-range-copy, +performance-move-const-arg, +performance-unnecessary-copy-initialization, +readability-redundant-declaration, +readability-redundant-string-init, +' +CheckOptions: + - key: performance-move-const-arg.CheckTriviallyCopyableMove + value: false diff --git a/src/Makefile.am b/src/Makefile.am index 407fdf5a8fd4..e73245daeb38 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -6,34 +6,31 @@ print-%: FORCE @echo '$*'='$($*)' -DIST_SUBDIRS = secp256k1 univalue +DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) -AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) -AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) $(CORE_LDFLAGS) +AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) $(CORE_CXXFLAGS) +AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) $(CORE_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = -if EMBEDDED_UNIVALUE -LIBUNIVALUE = univalue/libunivalue.la +lib_LTLIBRARIES = +noinst_LTLIBRARIES = -$(LIBUNIVALUE): $(wildcard univalue/lib/*) $(wildcard univalue/include/*) - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) -else -LIBUNIVALUE = $(UNIVALUE_LIBS) -endif - -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) +bin_PROGRAMS = +noinst_PROGRAMS = +TESTS = +BENCHMARKS = -BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(LEVELDB_CPPFLAGS) -LIBBITCOIN_SERVER=libbitcoin_server.a +LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libbitcoin_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a LIBBITCOIN_UTIL=libbitcoin_util.a -LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.a +LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.la LIBBITCOINQT=qt/libbitcoinqt.a LIBSECP256K1=secp256k1/libsecp256k1.la @@ -43,24 +40,32 @@ endif if BUILD_BITCOIN_LIBS LIBBITCOINCONSENSUS=libbitcoinconsensus.la endif +if BUILD_BITCOIN_KERNEL_LIB +LIBBITCOINKERNEL=libbitcoinkernel.la +endif if ENABLE_WALLET LIBBITCOIN_WALLET=libbitcoin_wallet.a LIBBITCOIN_WALLET_TOOL=libbitcoin_wallet_tool.a endif -LIBBITCOIN_CRYPTO= $(LIBBITCOIN_CRYPTO_BASE) +LIBBITCOIN_CRYPTO = $(LIBBITCOIN_CRYPTO_BASE) if ENABLE_SSE41 -LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.a +LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SSE41) endif if ENABLE_AVX2 -LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.a +LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) endif -if ENABLE_SHANI -LIBBITCOIN_CRYPTO_SHANI = crypto/libbitcoin_crypto_shani.a -LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SHANI) +if ENABLE_X86_SHANI +LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.la +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_X86_SHANI) +endif +if ENABLE_ARM_SHANI +LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.la +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_SHANI) endif +noinst_LTLIBRARIES += $(LIBBITCOIN_CRYPTO) $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) @@ -68,24 +73,16 @@ $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(w # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: EXTRA_LIBRARIES += \ - $(LIBBITCOIN_CRYPTO) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_CLI) \ $(LIBBITCOIN_IPC) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_WALLET_TOOL) \ $(LIBBITCOIN_ZMQ) -lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) - -bin_PROGRAMS = -noinst_PROGRAMS = -TESTS = -BENCHMARKS = - if BUILD_BITCOIND bin_PROGRAMS += bitcoind endif @@ -112,18 +109,22 @@ if BUILD_BITCOIN_UTIL bin_PROGRAMS += bitcoin-util endif +if BUILD_BITCOIN_CHAINSTATE + bin_PROGRAMS += bitcoin-chainstate +endif + .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ addrdb.h \ addrman.h \ + addrman_impl.h \ attributes.h \ banman.h \ base58.h \ bech32.h \ blockencodings.h \ blockfilter.h \ - bloom.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -131,12 +132,14 @@ BITCOIN_CORE_H = \ checkqueue.h \ clientversion.h \ coins.h \ - compat.h \ + common/bloom.h \ + common/run_command.h \ + common/url.h \ compat/assumptions.h \ compat/byteswap.h \ + compat/compat.h \ compat/cpuid.h \ compat/endian.h \ - compat/sanity.h \ compressor.h \ consensus/consensus.h \ consensus/tx_check.h \ @@ -150,6 +153,7 @@ BITCOIN_CORE_H = \ external_signer.h \ flatfile.h \ fs.h \ + headerssync.h \ httprpc.h \ httpserver.h \ i2p.h \ @@ -168,6 +172,16 @@ BITCOIN_CORE_H = \ interfaces/ipc.h \ interfaces/node.h \ interfaces/wallet.h \ + kernel/chain.h \ + kernel/chainstatemanager_opts.h \ + kernel/checks.h \ + kernel/coinstats.h \ + kernel/context.h \ + kernel/mempool_entry.h \ + kernel/mempool_limits.h \ + kernel/mempool_options.h \ + kernel/mempool_persist.h \ + kernel/validation_cache_sizes.h \ key.h \ key_io.h \ logging.h \ @@ -175,26 +189,37 @@ BITCOIN_CORE_H = \ mapport.h \ memusage.h \ merkleblock.h \ - miner.h \ net.h \ net_permissions.h \ net_processing.h \ net_types.h \ netaddress.h \ netbase.h \ + netgroup.h \ netmessagemaker.h \ node/blockstorage.h \ + node/caches.h \ + node/chainstate.h \ + node/chainstatemanager_args.h \ node/coin.h \ - node/coinstats.h \ + node/connection_types.h \ node/context.h \ + node/eviction.h \ + node/interface_ui.h \ + node/mempool_args.h \ + node/mempool_persist_args.h \ + node/miner.h \ + node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ - node/ui_interface.h \ + node/txreconciliation.h \ node/utxo_snapshot.h \ + node/validation_cache_args.h \ noui.h \ outputtype.h \ policy/feerate.h \ policy/fees.h \ + policy/fees_args.h \ policy/packages.h \ policy/policy.h \ policy/rbf.h \ @@ -204,20 +229,23 @@ BITCOIN_CORE_H = \ psbt.h \ random.h \ randomenv.h \ + rest.h \ reverse_iterator.h \ rpc/blockchain.h \ rpc/client.h \ + rpc/mempool.h \ rpc/mining.h \ - rpc/net.h \ rpc/protocol.h \ rpc/rawtransaction_util.h \ rpc/register.h \ rpc/request.h \ rpc/server.h \ + rpc/server_util.h \ rpc/util.h \ scheduler.h \ script/descriptor.h \ script/keyorigin.h \ + script/miniscript.h \ script/sigcache.h \ script/sign.h \ script/signingprovider.h \ @@ -231,7 +259,6 @@ BITCOIN_CORE_H = \ support/events.h \ support/lockedpool.h \ sync.h \ - threadinterrupt.h \ threadsafety.h \ timedata.h \ torcontrol.h \ @@ -242,10 +269,12 @@ BITCOIN_CORE_H = \ undo.h \ util/asmap.h \ util/bip32.h \ + util/bitdeque.h \ util/bytevectorhash.h \ util/check.h \ util/epochguard.h \ util/error.h \ + util/fastrange.h \ util/fees.h \ util/getuniquepath.h \ util/golombrice.h \ @@ -254,22 +283,28 @@ BITCOIN_CORE_H = \ util/macros.h \ util/message.h \ util/moneystr.h \ + util/overflow.h \ + util/overloaded.h \ util/rbf.h \ util/readwritefile.h \ + util/result.h \ util/serfloat.h \ util/settings.h \ util/sock.h \ util/spanparsing.h \ util/string.h \ + util/syscall_sandbox.h \ + util/syserror.h \ util/system.h \ util/thread.h \ + util/threadinterrupt.h \ util/threadnames.h \ util/time.h \ util/tokenpipe.h \ util/trace.h \ util/translation.h \ + util/types.h \ util/ui_change_type.h \ - util/url.h \ util/vector.h \ validation.h \ validationinterface.h \ @@ -287,7 +322,8 @@ BITCOIN_CORE_H = \ wallet/ismine.h \ wallet/load.h \ wallet/receive.h \ - wallet/rpcwallet.h \ + wallet/rpc/util.h \ + wallet/rpc/wallet.h \ wallet/salvage.h \ wallet/scriptpubkeyman.h \ wallet/spend.h \ @@ -312,15 +348,10 @@ obj/build.h: FORCE "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h -ipc/capnp/libbitcoin_ipc_a-ipc.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) - -# server: shared between bitcoind and bitcoin-qt -# Contains code accessing mempool and chain state that is meant to be separated -# from wallet and gui code (see node/README.md). Shared code should go in -# libbitcoin_common or libbitcoin_util libraries, instead. -libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -libbitcoin_server_a_SOURCES = \ +# node # +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +libbitcoin_node_a_SOURCES = \ addrdb.cpp \ addrman.cpp \ banman.cpp \ @@ -331,6 +362,7 @@ libbitcoin_server_a_SOURCES = \ dbwrapper.cpp \ deploymentstatus.cpp \ flatfile.cpp \ + headerssync.cpp \ httprpc.cpp \ httpserver.cpp \ i2p.cpp \ @@ -339,31 +371,54 @@ libbitcoin_server_a_SOURCES = \ index/coinstatsindex.cpp \ index/txindex.cpp \ init.cpp \ + kernel/chain.cpp \ + kernel/checks.cpp \ + kernel/coinstats.cpp \ + kernel/context.cpp \ + kernel/mempool_persist.cpp \ mapport.cpp \ - miner.cpp \ net.cpp \ net_processing.cpp \ + netgroup.cpp \ node/blockstorage.cpp \ + node/caches.cpp \ + node/chainstate.cpp \ + node/chainstatemanager_args.cpp \ node/coin.cpp \ - node/coinstats.cpp \ + node/connection_types.cpp \ node/context.cpp \ + node/eviction.cpp \ + node/interface_ui.cpp \ node/interfaces.cpp \ + node/mempool_args.cpp \ + node/mempool_persist_args.cpp \ + node/miner.cpp \ + node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ - node/ui_interface.cpp \ + node/txreconciliation.cpp \ + node/utxo_snapshot.cpp \ + node/validation_cache_args.cpp \ noui.cpp \ policy/fees.cpp \ + policy/fees_args.cpp \ policy/packages.cpp \ policy/rbf.cpp \ policy/settings.cpp \ pow.cpp \ rest.cpp \ rpc/blockchain.cpp \ + rpc/fees.cpp \ + rpc/mempool.cpp \ rpc/mining.cpp \ - rpc/misc.cpp \ rpc/net.cpp \ + rpc/node.cpp \ + rpc/output_script.cpp \ rpc/rawtransaction.cpp \ rpc/server.cpp \ + rpc/server_util.cpp \ + rpc/signmessage.cpp \ + rpc/txoutproof.cpp \ script/sigcache.cpp \ shutdown.cpp \ signet.cpp \ @@ -379,12 +434,15 @@ libbitcoin_server_a_SOURCES = \ $(BITCOIN_CORE_H) if ENABLE_WALLET -libbitcoin_server_a_SOURCES += wallet/init.cpp +libbitcoin_node_a_SOURCES += wallet/init.cpp +libbitcoin_node_a_CPPFLAGS += $(BDB_CPPFLAGS) endif if !ENABLE_WALLET -libbitcoin_server_a_SOURCES += dummywallet.cpp +libbitcoin_node_a_SOURCES += dummywallet.cpp endif +# +# zmq # if ENABLE_ZMQ libbitcoin_zmq_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -395,11 +453,10 @@ libbitcoin_zmq_a_SOURCES = \ zmq/zmqrpc.cpp \ zmq/zmqutil.cpp endif +# - -# wallet: shared between bitcoind and bitcoin-qt, but only linked -# when wallet enabled -libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(SQLITE_CFLAGS) +# wallet # +libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(BDB_CPPFLAGS) $(SQLITE_CFLAGS) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ wallet/coincontrol.cpp \ @@ -413,8 +470,15 @@ libbitcoin_wallet_a_SOURCES = \ wallet/interfaces.cpp \ wallet/load.cpp \ wallet/receive.cpp \ - wallet/rpcdump.cpp \ - wallet/rpcwallet.cpp \ + wallet/rpc/addresses.cpp \ + wallet/rpc/backup.cpp \ + wallet/rpc/coins.cpp \ + wallet/rpc/encrypt.cpp \ + wallet/rpc/spend.cpp \ + wallet/rpc/signmessage.cpp \ + wallet/rpc/transactions.cpp \ + wallet/rpc/util.cpp \ + wallet/rpc/wallet.cpp \ wallet/scriptpubkeyman.cpp \ wallet/spend.cpp \ wallet/transaction.cpp \ @@ -430,17 +494,27 @@ endif if USE_BDB libbitcoin_wallet_a_SOURCES += wallet/bdb.cpp wallet/salvage.cpp endif +# -libbitcoin_wallet_tool_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +# wallet tool # +libbitcoin_wallet_tool_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_wallet_tool_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_tool_a_SOURCES = \ wallet/wallettool.cpp \ $(BITCOIN_CORE_H) +# + +# crypto # +crypto_libbitcoin_crypto_base_la_CPPFLAGS = $(AM_CPPFLAGS) -# crypto primitives library -crypto_libbitcoin_crypto_base_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_base_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_base_a_SOURCES = \ +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +crypto_libbitcoin_crypto_base_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_base_la_LDFLAGS = $(AM_LDFLAGS) -static + +crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ crypto/chacha_poly_aead.h \ @@ -472,34 +546,53 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/siphash.h if USE_ASM -crypto_libbitcoin_crypto_base_a_SOURCES += crypto/sha256_sse4.cpp +crypto_libbitcoin_crypto_base_la_SOURCES += crypto/sha256_sse4.cpp endif -crypto_libbitcoin_crypto_sse41_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_sse41_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_sse41_a_CXXFLAGS += $(SSE41_CXXFLAGS) -crypto_libbitcoin_crypto_sse41_a_CPPFLAGS += -DENABLE_SSE41 -crypto_libbitcoin_crypto_sse41_a_SOURCES = crypto/sha256_sse41.cpp - -crypto_libbitcoin_crypto_avx2_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_avx2_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_avx2_a_CXXFLAGS += $(AVX2_CXXFLAGS) -crypto_libbitcoin_crypto_avx2_a_CPPFLAGS += -DENABLE_AVX2 -crypto_libbitcoin_crypto_avx2_a_SOURCES = crypto/sha256_avx2.cpp - -crypto_libbitcoin_crypto_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_shani_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_shani_a_CXXFLAGS += $(SHANI_CXXFLAGS) -crypto_libbitcoin_crypto_shani_a_CPPFLAGS += -DENABLE_SHANI -crypto_libbitcoin_crypto_shani_a_SOURCES = crypto/sha256_shani.cpp +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_sse41_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_sse41_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_sse41_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_sse41_la_CXXFLAGS += $(SSE41_CXXFLAGS) +crypto_libbitcoin_crypto_sse41_la_CPPFLAGS += -DENABLE_SSE41 +crypto_libbitcoin_crypto_sse41_la_SOURCES = crypto/sha256_sse41.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_avx2_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_avx2_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_avx2_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_avx2_la_CXXFLAGS += $(AVX2_CXXFLAGS) +crypto_libbitcoin_crypto_avx2_la_CPPFLAGS += -DENABLE_AVX2 +crypto_libbitcoin_crypto_avx2_la_SOURCES = crypto/sha256_avx2.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_x86_shani_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_x86_shani_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_x86_shani_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_x86_shani_la_CXXFLAGS += $(X86_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_x86_shani_la_CPPFLAGS += -DENABLE_X86_SHANI +crypto_libbitcoin_crypto_x86_shani_la_SOURCES = crypto/sha256_x86_shani.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_arm_shani_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS += -DENABLE_ARM_SHANI +crypto_libbitcoin_crypto_arm_shani_la_SOURCES = crypto/sha256_arm_shani.cpp +# -# consensus: shared between all executables that validate any consensus rules. +# consensus # libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_consensus_a_SOURCES = \ - amount.h \ arith_uint256.cpp \ arith_uint256.h \ + consensus/amount.h \ consensus/merkle.cpp \ consensus/merkle.h \ consensus/params.h \ @@ -529,16 +622,19 @@ libbitcoin_consensus_a_SOURCES = \ util/strencodings.cpp \ util/strencodings.h \ version.h +# -# common: shared between bitcoind, and bitcoin-qt and non-server tools -libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +# common # +libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ base58.cpp \ bech32.cpp \ - bloom.cpp \ chainparams.cpp \ coins.cpp \ + common/bloom.cpp \ + common/interfaces.cpp \ + common/run_command.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -548,6 +644,7 @@ libbitcoin_common_a_SOURCES = \ key.cpp \ key_io.cpp \ merkleblock.cpp \ + net_types.cpp \ netaddress.cpp \ netbase.cpp \ net_permissions.cpp \ @@ -561,42 +658,43 @@ libbitcoin_common_a_SOURCES = \ rpc/util.cpp \ scheduler.cpp \ script/descriptor.cpp \ + script/miniscript.cpp \ script/sign.cpp \ script/signingprovider.cpp \ script/standard.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) -# util: shared between all executables. -# This library *must* be included to make sure that the glibc -# backward-compatibility objects and their sanity checks are linked. +if USE_LIBEVENT +libbitcoin_common_a_CPPFLAGS += $(EVENT_CFLAGS) +libbitcoin_common_a_SOURCES += common/url.cpp +endif +# + +# util # libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ - compat/glibcxx_sanity.cpp \ - compat/strnlen.cpp \ fs.cpp \ - interfaces/echo.cpp \ - interfaces/handler.cpp \ - interfaces/init.cpp \ logging.cpp \ random.cpp \ randomenv.cpp \ rpc/request.cpp \ support/cleanse.cpp \ sync.cpp \ - threadinterrupt.cpp \ util/asmap.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ + util/check.cpp \ util/error.cpp \ util/fees.cpp \ util/getuniquepath.cpp \ util/hasher.cpp \ util/sock.cpp \ + util/syserror.cpp \ util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ @@ -604,25 +702,19 @@ libbitcoin_util_a_SOURCES = \ util/readwritefile.cpp \ util/settings.cpp \ util/thread.cpp \ + util/threadinterrupt.cpp \ util/threadnames.cpp \ util/serfloat.cpp \ util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ + util/syscall_sandbox.cpp \ util/time.cpp \ util/tokenpipe.cpp \ $(BITCOIN_CORE_H) +# -if USE_LIBEVENT -libbitcoin_util_a_SOURCES += util/url.cpp -endif - -if GLIBC_BACK_COMPAT -libbitcoin_util_a_SOURCES += compat/glibc_compat.cpp -AM_LDFLAGS += $(COMPAT_LDFLAGS) -endif - -# cli: shared between bitcoin-cli and bitcoin-qt +# cli # libbitcoin_cli_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_cli_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_cli_a_SOURCES = \ @@ -653,23 +745,22 @@ bitcoin_bin_ldadd = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) -bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) +bitcoin_bin_ldadd += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) bitcoind_SOURCES = $(bitcoin_daemon_sources) init/bitcoind.cpp bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoind_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoind_LDFLAGS = $(bitcoin_bin_ldflags) -bitcoind_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) +bitcoind_LDADD = $(LIBBITCOIN_NODE) $(bitcoin_bin_ldadd) bitcoin_node_SOURCES = $(bitcoin_daemon_sources) init/bitcoin-node.cpp bitcoin_node_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoin_node_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoin_node_LDFLAGS = $(bitcoin_bin_ldflags) -bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS) +bitcoin_node_LDADD = $(LIBBITCOIN_NODE) $(bitcoin_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp @@ -684,10 +775,11 @@ endif bitcoin_cli_LDADD = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ + $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) -bitcoin_cli_LDADD += $(BOOST_LIBS) $(EVENT_LIBS) +bitcoin_cli_LDADD += $(EVENT_LIBS) # # bitcoin-tx binary # @@ -707,16 +799,25 @@ bitcoin_tx_LDADD = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) - -bitcoin_tx_LDADD += $(BOOST_LIBS) # # bitcoin-wallet binary # bitcoin_wallet_SOURCES = bitcoin-wallet.cpp +bitcoin_wallet_SOURCES += init/bitcoin-wallet.cpp bitcoin_wallet_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoin_wallet_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoin_wallet_LDFLAGS = $(bitcoin_bin_ldflags) -bitcoin_wallet_LDADD = $(LIBBITCOIN_WALLET_TOOL) $(bitcoin_bin_ldadd) +bitcoin_wallet_LDADD = \ + $(LIBBITCOIN_WALLET_TOOL) \ + $(LIBBITCOIN_WALLET) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBUNIVALUE) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBSECP256K1) \ + $(BDB_LIBS) \ + $(SQLITE_LIBS) if TARGET_WINDOWS bitcoin_wallet_SOURCES += bitcoin-wallet-res.rc @@ -740,18 +841,137 @@ bitcoin_util_LDADD = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) +# + +# bitcoin-chainstate binary # +bitcoin_chainstate_SOURCES = bitcoin-chainstate.cpp +bitcoin_chainstate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) +bitcoin_chainstate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) + +# $(LIBTOOL_APP_LDFLAGS) deliberately omitted here so that we can test linking +# bitcoin-chainstate against libbitcoinkernel as a shared or static library by +# setting --{en,dis}able-shared. +bitcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(PTHREAD_FLAGS) +bitcoin_chainstate_LDADD = $(LIBBITCOINKERNEL) +# + +# bitcoinkernel library # +if BUILD_BITCOIN_KERNEL_LIB +lib_LTLIBRARIES += $(LIBBITCOINKERNEL) + +libbitcoinkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS) +libbitcoinkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) +libbitcoinkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) + +# libbitcoinkernel requires default symbol visibility, explicitly specify that +# here so that things still work even when user configures with +# --enable-reduce-exports +# +# Note this is a quick hack that will be removed as we incrementally define what +# to export from the library. +libbitcoinkernel_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -fvisibility=default + +# TODO: For now, Specify -static in both CXXFLAGS and LDFLAGS when building for +# windows targets so libtool will only build a static version of this +# library. There are unresolved problems when building dll's for mingw-w64 +# and attempting to statically embed libstdc++, libpthread, etc. +if TARGET_WINDOWS +libbitcoinkernel_la_LDFLAGS += -static +libbitcoinkernel_la_CXXFLAGS += -static +endif + +# TODO: libbitcoinkernel is a work in progress consensus engine library, as more +# and more modules are decoupled from the consensus engine, this list will +# shrink to only those which are absolutely necessary. +libbitcoinkernel_la_SOURCES = \ + kernel/bitcoinkernel.cpp \ + arith_uint256.cpp \ + chain.cpp \ + chainparamsbase.cpp \ + chainparams.cpp \ + clientversion.cpp \ + coins.cpp \ + compressor.cpp \ + consensus/merkle.cpp \ + consensus/tx_check.cpp \ + consensus/tx_verify.cpp \ + core_read.cpp \ + dbwrapper.cpp \ + deploymentinfo.cpp \ + deploymentstatus.cpp \ + flatfile.cpp \ + fs.cpp \ + hash.cpp \ + kernel/chain.cpp \ + kernel/checks.cpp \ + kernel/coinstats.cpp \ + kernel/context.cpp \ + kernel/mempool_persist.cpp \ + key.cpp \ + logging.cpp \ + node/blockstorage.cpp \ + node/chainstate.cpp \ + node/interface_ui.cpp \ + node/utxo_snapshot.cpp \ + policy/feerate.cpp \ + policy/fees.cpp \ + policy/packages.cpp \ + policy/policy.cpp \ + policy/rbf.cpp \ + policy/settings.cpp \ + pow.cpp \ + primitives/block.cpp \ + primitives/transaction.cpp \ + pubkey.cpp \ + random.cpp \ + randomenv.cpp \ + scheduler.cpp \ + script/interpreter.cpp \ + script/script.cpp \ + script/script_error.cpp \ + script/sigcache.cpp \ + script/standard.cpp \ + shutdown.cpp \ + signet.cpp \ + support/cleanse.cpp \ + support/lockedpool.cpp \ + sync.cpp \ + txdb.cpp \ + txmempool.cpp \ + uint256.cpp \ + util/check.cpp \ + util/getuniquepath.cpp \ + util/hasher.cpp \ + util/moneystr.cpp \ + util/rbf.cpp \ + util/serfloat.cpp \ + util/settings.cpp \ + util/strencodings.cpp \ + util/string.cpp \ + util/syscall_sandbox.cpp \ + util/syserror.cpp \ + util/system.cpp \ + util/thread.cpp \ + util/threadnames.cpp \ + util/time.cpp \ + util/tokenpipe.cpp \ + validation.cpp \ + validationinterface.cpp \ + versionbits.cpp \ + warnings.cpp -bitcoin_util_LDADD += $(BOOST_LIBS) +# Required for obj/build.h to be generated first. +# More details: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html +libbitcoinkernel_la-clientversion.l$(OBJEXT): obj/build.h +endif # BUILD_BITCOIN_KERNEL_LIB # # bitcoinconsensus library # if BUILD_BITCOIN_LIBS -include_HEADERS = script/bitcoinconsensus.h -libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) +lib_LTLIBRARIES += $(LIBBITCOINCONSENSUS) -if GLIBC_BACK_COMPAT - libbitcoinconsensus_la_SOURCES += compat/glibc_compat.cpp -endif +include_HEADERS = script/bitcoinconsensus.h +libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_la_SOURCES) $(libbitcoin_consensus_a_SOURCES) libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) @@ -801,7 +1021,6 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) clean-local: -$(MAKE) -C secp256k1 clean - -$(MAKE) -C univalue clean -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -f config.h -rm -rf test/__pycache__ @@ -812,20 +1031,8 @@ clean-local: $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ check-symbols: $(bin_PROGRAMS) -if TARGET_DARWIN - @echo "Checking macOS dynamic libraries..." + @echo "Running symbol and dynamic library checks..." $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if TARGET_WINDOWS - @echo "Checking Windows dynamic libraries..." - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if TARGET_LINUX - @echo "Checking glibc back compat..." - $(AM_V_at) CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif check-security: $(bin_PROGRAMS) if HARDEN @@ -839,12 +1046,18 @@ libbitcoin_ipc_mpgen_input = \ EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) %.capnp: +# Explicitly list dependencies on generated headers as described in +# https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually +ipc/capnp/libbitcoin_ipc_a-protocol.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) + if BUILD_MULTIPROCESS LIBBITCOIN_IPC=libbitcoin_ipc.a libbitcoin_ipc_a_SOURCES = \ + ipc/capnp/context.h \ ipc/capnp/init-types.h \ ipc/capnp/protocol.cpp \ ipc/capnp/protocol.h \ + ipc/context.h \ ipc/exception.h \ ipc/interfaces.cpp \ ipc/process.cpp \ @@ -866,17 +1079,24 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output) CLEANFILES += $(libbitcoin_ipc_mpgen_output) endif -if EMBEDDED_LEVELDB +%.raw.h: %.raw + @$(MKDIR_P) $(@D) + @{ \ + echo "static unsigned const char $(*F)_raw[] = {" && \ + $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ + echo "};"; \ + } > "$@.new" && mv -f "$@.new" "$@" + @echo "Generated $@" + +include Makefile.minisketch.include + include Makefile.crc32c.include include Makefile.leveldb.include -endif include Makefile.test_util.include include Makefile.test_fuzz.include -if ENABLE_TESTS include Makefile.test.include -endif if ENABLE_BENCH include Makefile.bench.include @@ -889,3 +1109,5 @@ endif if ENABLE_QT_TESTS include Makefile.qttest.include endif + +include Makefile.univalue.include diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 2a8e4a0aaca0..f1e4e706a1fe 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -13,58 +13,64 @@ GENERATED_BENCH_FILES = $(RAW_BENCH_FILES:.raw=.raw.h) bench_bench_bitcoin_SOURCES = \ $(RAW_BENCH_FILES) \ bench/addrman.cpp \ - bench/bench_bitcoin.cpp \ + bench/base58.cpp \ + bench/bech32.cpp \ bench/bench.cpp \ bench/bench.h \ + bench/bench_bitcoin.cpp \ bench/block_assemble.cpp \ + bench/ccoins_caching.cpp \ + bench/chacha20.cpp \ + bench/chacha_poly_aead.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ - bench/data.h \ + bench/crypto_hash.cpp \ bench/data.cpp \ + bench/data.h \ + bench/descriptors.cpp \ bench/duplicate_inputs.cpp \ bench/examples.cpp \ - bench/rollingbloom.cpp \ - bench/chacha20.cpp \ - bench/chacha_poly_aead.cpp \ - bench/crypto_hash.cpp \ - bench/ccoins_caching.cpp \ bench/gcs_filter.cpp \ bench/hashpadding.cpp \ - bench/merkle_root.cpp \ + bench/load_external.cpp \ + bench/lockedpool.cpp \ + bench/logging.cpp \ bench/mempool_eviction.cpp \ bench/mempool_stress.cpp \ - bench/nanobench.h \ + bench/merkle_root.cpp \ bench/nanobench.cpp \ + bench/nanobench.h \ bench/peer_eviction.cpp \ + bench/poly1305.cpp \ + bench/prevector.cpp \ + bench/rollingbloom.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ + bench/strencodings.cpp \ bench/util_time.cpp \ - bench/verify_script.cpp \ - bench/base58.cpp \ - bench/bech32.cpp \ - bench/lockedpool.cpp \ - bench/poly1305.cpp \ - bench/prevector.cpp + bench/verify_script.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) -bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ +bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) bench_bench_bitcoin_LDADD = \ - $(LIBBITCOIN_SERVER) \ + $(LIBTEST_UTIL) \ + $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ - $(LIBTEST_UTIL) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(LIBUNIVALUE) \ $(EVENT_PTHREADS_LIBS) \ - $(EVENT_LIBS) + $(EVENT_LIBS) \ + $(MINIUPNPC_LIBS) \ + $(NATPMP_LIBS) if ENABLE_ZMQ bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) @@ -73,11 +79,11 @@ endif if ENABLE_WALLET bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp +bench_bench_bitcoin_SOURCES += bench/wallet_loading.cpp +bench_bench_bitcoin_SOURCES += bench/wallet_create_tx.cpp +bench_bench_bitcoin_LDADD += $(BDB_LIBS) $(SQLITE_LIBS) endif -bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) -bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) - CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) CLEANFILES += $(CLEAN_BITCOIN_BENCH) @@ -91,12 +97,3 @@ bench: $(BENCH_BINARY) FORCE bitcoin_bench_clean : FORCE rm -f $(CLEAN_BITCOIN_BENCH) $(bench_bench_bitcoin_OBJECTS) $(BENCH_BINARY) - -%.raw.h: %.raw - @$(MKDIR_P) $(@D) - @{ \ - echo "static unsigned const char $(*F)_raw[] = {" && \ - $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ - echo "};"; \ - } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" diff --git a/src/Makefile.crc32c.include b/src/Makefile.crc32c.include index 113272e65e89..c4dd84991d27 100644 --- a/src/Makefile.crc32c.include +++ b/src/Makefile.crc32c.include @@ -2,10 +2,9 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -LIBCRC32C_INT = crc32c/libcrc32c.a -LIBLEVELDB_SSE42_INT = leveldb/libleveldb_sse42.a +LIBCRC32C_INT = crc32c/libcrc32c.la -EXTRA_LIBRARIES += $(LIBCRC32C_INT) +noinst_LTLIBRARIES += $(LIBCRC32C_INT) LIBCRC32C = $(LIBCRC32C_INT) @@ -14,7 +13,6 @@ CRC32C_CPPFLAGS_INT += -I$(srcdir)/crc32c/include CRC32C_CPPFLAGS_INT += -DHAVE_BUILTIN_PREFETCH=@HAVE_BUILTIN_PREFETCH@ CRC32C_CPPFLAGS_INT += -DHAVE_MM_PREFETCH=@HAVE_MM_PREFETCH@ CRC32C_CPPFLAGS_INT += -DHAVE_STRONG_GETAUXVAL=@HAVE_STRONG_GETAUXVAL@ -CRC32C_CPPFLAGS_INT += -DHAVE_WEAK_GETAUXVAL=@HAVE_WEAK_GETAUXVAL@ CRC32C_CPPFLAGS_INT += -DCRC32C_TESTS_BUILT_WITH_GLOG=0 if ENABLE_SSE42 @@ -35,41 +33,49 @@ else CRC32C_CPPFLAGS_INT += -DBYTE_ORDER_BIG_ENDIAN=0 endif -crc32c_libcrc32c_a_CPPFLAGS = $(AM_CPPFLAGS) $(CRC32C_CPPFLAGS_INT) $(CRC32C_CPPFLAGS) -crc32c_libcrc32c_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) - -crc32c_libcrc32c_a_SOURCES = -crc32c_libcrc32c_a_SOURCES += crc32c/include/crc32c/crc32c.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64_check.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_internal.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_prefetch.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_read_le.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_round_up.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_sse42_check.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_sse42.h - -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c.cc -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_portable.cc +crc32c_libcrc32c_la_CPPFLAGS = $(AM_CPPFLAGS) $(CRC32C_CPPFLAGS_INT) $(CRC32C_CPPFLAGS) + +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +crc32c_libcrc32c_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crc32c_libcrc32c_la_LDFLAGS = $(AM_LDFLAGS) -static + +crc32c_libcrc32c_la_SOURCES = +crc32c_libcrc32c_la_SOURCES += crc32c/include/crc32c/crc32c.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_arm64.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_arm64_check.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_internal.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_prefetch.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_read_le.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_round_up.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_sse42_check.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_sse42.h + +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c.cc +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_portable.cc if ENABLE_SSE42 -LIBCRC32C_SSE42_INT = crc32c/libcrc32c_sse42.a -EXTRA_LIBRARIES += $(LIBCRC32C_SSE42_INT) +LIBCRC32C_SSE42_INT = crc32c/libcrc32c_sse42.la +noinst_LTLIBRARIES += $(LIBCRC32C_SSE42_INT) LIBCRC32C += $(LIBCRC32C_SSE42_INT) -crc32c_libcrc32c_sse42_a_CPPFLAGS = $(crc32c_libcrc32c_a_CPPFLAGS) -crc32c_libcrc32c_sse42_a_CXXFLAGS = $(crc32c_libcrc32c_a_CXXFLAGS) $(SSE42_CXXFLAGS) +crc32c_libcrc32c_sse42_la_CPPFLAGS = $(crc32c_libcrc32c_la_CPPFLAGS) +crc32c_libcrc32c_sse42_la_CXXFLAGS = $(crc32c_libcrc32c_la_CXXFLAGS) $(SSE42_CXXFLAGS) +crc32c_libcrc32c_sse42_la_LDFLAGS = $(crc32c_libcrc32c_la_LDFLAGS) -crc32c_libcrc32c_sse42_a_SOURCES = crc32c/src/crc32c_sse42.cc +crc32c_libcrc32c_sse42_la_SOURCES = crc32c/src/crc32c_sse42.cc endif if ENABLE_ARM_CRC -LIBCRC32C_ARM_CRC_INT = crc32c/libcrc32c_arm_crc.a -EXTRA_LIBRARIES += $(LIBCRC32C_ARM_CRC_INT) +LIBCRC32C_ARM_CRC_INT = crc32c/libcrc32c_arm_crc.la +noinst_LTLIBRARIES += $(LIBCRC32C_ARM_CRC_INT) LIBCRC32C += $(LIBCRC32C_ARM_CRC_INT) -crc32c_libcrc32c_arm_crc_a_CPPFLAGS = $(crc32c_libcrc32c_a_CPPFLAGS) -crc32c_libcrc32c_arm_crc_a_CXXFLAGS = $(crc32c_libcrc32c_a_CXXFLAGS) $(ARM_CRC_CXXFLAGS) +crc32c_libcrc32c_arm_crc_la_CPPFLAGS = $(crc32c_libcrc32c_la_CPPFLAGS) +crc32c_libcrc32c_arm_crc_la_CXXFLAGS = $(crc32c_libcrc32c_la_CXXFLAGS) $(ARM_CRC_CXXFLAGS) +crc32c_libcrc32c_arm_crc_la_LDFLAGS = $(crc32c_libcrc32c_la_LDFLAGS) -crc32c_libcrc32c_arm_crc_a_SOURCES = crc32c/src/crc32c_arm64.cc +crc32c_libcrc32c_arm_crc_la_SOURCES = crc32c/src/crc32c_arm64.cc endif diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index ce1f93f11f3b..bf14fe206b92 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -2,17 +2,17 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -LIBLEVELDB_INT = leveldb/libleveldb.a -LIBMEMENV_INT = leveldb/libmemenv.a +LIBLEVELDB_INT = leveldb/libleveldb.la +LIBMEMENV_INT = leveldb/libmemenv.la -EXTRA_LIBRARIES += $(LIBLEVELDB_INT) -EXTRA_LIBRARIES += $(LIBMEMENV_INT) +noinst_LTLIBRARIES += $(LIBLEVELDB_INT) +noinst_LTLIBRARIES += $(LIBMEMENV_INT) -LIBLEVELDB += $(LIBLEVELDB_INT) $(LIBCRC32C) -LIBMEMENV += $(LIBMEMENV_INT) +LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C) +LIBMEMENV = $(LIBMEMENV_INT) +LEVELDB_CPPFLAGS = LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include -LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv LEVELDB_CPPFLAGS_INT = LEVELDB_CPPFLAGS_INT += -I$(srcdir)/leveldb @@ -36,111 +36,118 @@ else LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX endif -leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) +leveldb_libleveldb_la_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_SOURCES= -leveldb_libleveldb_a_SOURCES += leveldb/port/port_stdcxx.h -leveldb_libleveldb_a_SOURCES += leveldb/port/port.h -leveldb_libleveldb_a_SOURCES += leveldb/port/thread_annotations.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/db.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/options.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/comparator.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/filter_policy.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/slice.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table_builder.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/env.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/export.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/c.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/iterator.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/cache.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/dumpfile.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/write_batch.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/status.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_format.h -leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.h -leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.h -leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch_internal.h -leveldb_libleveldb_a_SOURCES += leveldb/db/filename.h -leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.h -leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.h -leveldb_libleveldb_a_SOURCES += leveldb/db/builder.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.h -leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.h -leveldb_libleveldb_a_SOURCES += leveldb/db/skiplist.h -leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.h -leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.h -leveldb_libleveldb_a_SOURCES += leveldb/db/snapshot.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.h -leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.h -leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.h -leveldb_libleveldb_a_SOURCES += leveldb/table/block.h -leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.h -leveldb_libleveldb_a_SOURCES += leveldb/table/merger.h -leveldb_libleveldb_a_SOURCES += leveldb/table/format.h -leveldb_libleveldb_a_SOURCES += leveldb/table/iterator_wrapper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.h -leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix_test_helper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/env_windows_test_helper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/arena.h -leveldb_libleveldb_a_SOURCES += leveldb/util/random.h -leveldb_libleveldb_a_SOURCES += leveldb/util/posix_logger.h -leveldb_libleveldb_a_SOURCES += leveldb/util/hash.h -leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.h -leveldb_libleveldb_a_SOURCES += leveldb/util/coding.h -leveldb_libleveldb_a_SOURCES += leveldb/util/testutil.h -leveldb_libleveldb_a_SOURCES += leveldb/util/mutexlock.h -leveldb_libleveldb_a_SOURCES += leveldb/util/logging.h -leveldb_libleveldb_a_SOURCES += leveldb/util/no_destructor.h -leveldb_libleveldb_a_SOURCES += leveldb/util/testharness.h -leveldb_libleveldb_a_SOURCES += leveldb/util/windows_logger.h +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +leveldb_libleveldb_la_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) -static +leveldb_libleveldb_la_LDFLAGS = $(AM_LDFLAGS) -static -leveldb_libleveldb_a_SOURCES += leveldb/db/builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/c.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/dumpfile.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/filename.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/repair.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/block.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/format.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/iterator.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/merger.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/table_builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/table.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/arena.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/bloom.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/cache.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/coding.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/comparator.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/env.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/filter_policy.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/hash.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/logging.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/options.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/status.cc +leveldb_libleveldb_la_SOURCES= +leveldb_libleveldb_la_SOURCES += leveldb/port/port_stdcxx.h +leveldb_libleveldb_la_SOURCES += leveldb/port/port.h +leveldb_libleveldb_la_SOURCES += leveldb/port/thread_annotations.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/db.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/options.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/comparator.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/filter_policy.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/slice.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/table_builder.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/env.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/export.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/c.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/iterator.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/cache.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/dumpfile.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/table.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/write_batch.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/status.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_format.h +leveldb_libleveldb_la_SOURCES += leveldb/db/memtable.h +leveldb_libleveldb_la_SOURCES += leveldb/db/version_set.h +leveldb_libleveldb_la_SOURCES += leveldb/db/write_batch_internal.h +leveldb_libleveldb_la_SOURCES += leveldb/db/filename.h +leveldb_libleveldb_la_SOURCES += leveldb/db/version_edit.h +leveldb_libleveldb_la_SOURCES += leveldb/db/dbformat.h +leveldb_libleveldb_la_SOURCES += leveldb/db/builder.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_writer.h +leveldb_libleveldb_la_SOURCES += leveldb/db/db_iter.h +leveldb_libleveldb_la_SOURCES += leveldb/db/skiplist.h +leveldb_libleveldb_la_SOURCES += leveldb/db/db_impl.h +leveldb_libleveldb_la_SOURCES += leveldb/db/table_cache.h +leveldb_libleveldb_la_SOURCES += leveldb/db/snapshot.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_reader.h +leveldb_libleveldb_la_SOURCES += leveldb/table/filter_block.h +leveldb_libleveldb_la_SOURCES += leveldb/table/block_builder.h +leveldb_libleveldb_la_SOURCES += leveldb/table/block.h +leveldb_libleveldb_la_SOURCES += leveldb/table/two_level_iterator.h +leveldb_libleveldb_la_SOURCES += leveldb/table/merger.h +leveldb_libleveldb_la_SOURCES += leveldb/table/format.h +leveldb_libleveldb_la_SOURCES += leveldb/table/iterator_wrapper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/crc32c.h +leveldb_libleveldb_la_SOURCES += leveldb/util/env_posix_test_helper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/env_windows_test_helper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/arena.h +leveldb_libleveldb_la_SOURCES += leveldb/util/random.h +leveldb_libleveldb_la_SOURCES += leveldb/util/posix_logger.h +leveldb_libleveldb_la_SOURCES += leveldb/util/hash.h +leveldb_libleveldb_la_SOURCES += leveldb/util/histogram.h +leveldb_libleveldb_la_SOURCES += leveldb/util/coding.h +leveldb_libleveldb_la_SOURCES += leveldb/util/testutil.h +leveldb_libleveldb_la_SOURCES += leveldb/util/mutexlock.h +leveldb_libleveldb_la_SOURCES += leveldb/util/logging.h +leveldb_libleveldb_la_SOURCES += leveldb/util/no_destructor.h +leveldb_libleveldb_la_SOURCES += leveldb/util/testharness.h +leveldb_libleveldb_la_SOURCES += leveldb/util/windows_logger.h + +leveldb_libleveldb_la_SOURCES += leveldb/db/builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/c.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/dbformat.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/db_impl.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/db_iter.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/dumpfile.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/filename.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/log_reader.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/log_writer.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/memtable.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/repair.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/table_cache.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/version_edit.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/version_set.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/write_batch.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/block_builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/block.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/filter_block.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/format.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/iterator.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/merger.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/table_builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/table.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/two_level_iterator.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/arena.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/bloom.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/cache.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/coding.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/comparator.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/crc32c.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/filter_policy.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/hash.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/histogram.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/logging.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/options.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/status.cc if TARGET_WINDOWS -leveldb_libleveldb_a_SOURCES += leveldb/util/env_windows.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env_windows.cc else -leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env_posix.cc endif -leveldb_libmemenv_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS) -leveldb_libmemenv_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS) -leveldb_libmemenv_a_SOURCES = leveldb/helpers/memenv/memenv.cc -leveldb_libmemenv_a_SOURCES += leveldb/helpers/memenv/memenv.h +leveldb_libmemenv_la_CPPFLAGS = $(leveldb_libleveldb_la_CPPFLAGS) +leveldb_libmemenv_la_CXXFLAGS = $(leveldb_libleveldb_la_CXXFLAGS) +leveldb_libmemenv_la_LDFLAGS = $(leveldb_libleveldb_la_LDFLAGS) +leveldb_libmemenv_la_SOURCES = leveldb/helpers/memenv/memenv.cc +leveldb_libmemenv_la_SOURCES += leveldb/helpers/memenv/memenv.h diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include new file mode 100644 index 000000000000..b337f483498e --- /dev/null +++ b/src/Makefile.minisketch.include @@ -0,0 +1,43 @@ +include minisketch/sources.mk + +LIBMINISKETCH_CPPFLAGS= +LIBMINISKETCH_CPPFLAGS += -DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_32 + +LIBMINISKETCH = minisketch/libminisketch.a +MINISKETCH_LIBS = $(LIBMINISKETCH) + +if ENABLE_CLMUL +LIBMINISKETCH_CLMUL = minisketch/libminisketch_clmul.a +LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL +MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL) +endif + +if HAVE_CLZ +LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ +endif + +EXTRA_LIBRARIES += $(MINISKETCH_LIBS) + +minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT) +minisketch_libminisketch_clmul_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(CLMUL_CXXFLAGS) +minisketch_libminisketch_clmul_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) + +minisketch_libminisketch_a_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) $(MINISKETCH_LIB_SOURCES_INT) +minisketch_libminisketch_a_SOURCES += $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_DIST_HEADERS_INT) +minisketch_libminisketch_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) +minisketch_libminisketch_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) + +if ENABLE_TESTS +if !ENABLE_FUZZ +MINISKETCH_TEST = minisketch/test +TESTS += $(MINISKETCH_TEST) +noinst_PROGRAMS += $(MINISKETCH_TEST) + +minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) +minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) +minisketch_test_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +minisketch_test_LDADD = $(MINISKETCH_LIBS) +minisketch_test_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) + +endif +endif diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index a1821cafe351..602a1182598c 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -40,9 +40,9 @@ QT_MOC_CPP = \ qt/moc_askpassphrasedialog.cpp \ qt/moc_createwalletdialog.cpp \ qt/moc_bantablemodel.cpp \ + qt/moc_bitcoin.cpp \ qt/moc_bitcoinaddressvalidator.cpp \ qt/moc_bitcoinamountfield.cpp \ - qt/moc_bitcoin.cpp \ qt/moc_bitcoingui.cpp \ qt/moc_bitcoinunits.cpp \ qt/moc_clientmodel.cpp \ @@ -51,6 +51,7 @@ QT_MOC_CPP = \ qt/moc_csvmodelwriter.cpp \ qt/moc_editaddressdialog.cpp \ qt/moc_guiutil.cpp \ + qt/moc_initexecutor.cpp \ qt/moc_intro.cpp \ qt/moc_macdockiconhandler.cpp \ qt/moc_macnotificationhandler.cpp \ @@ -109,9 +110,9 @@ BITCOIN_QT_H = \ qt/addresstablemodel.h \ qt/askpassphrasedialog.h \ qt/bantablemodel.h \ + qt/bitcoin.h \ qt/bitcoinaddressvalidator.h \ qt/bitcoinamountfield.h \ - qt/bitcoin.h \ qt/bitcoingui.h \ qt/bitcoinunits.h \ qt/clientmodel.h \ @@ -122,6 +123,7 @@ BITCOIN_QT_H = \ qt/editaddressdialog.h \ qt/guiconstants.h \ qt/guiutil.h \ + qt/initexecutor.h \ qt/intro.h \ qt/macdockiconhandler.h \ qt/macnotificationhandler.h \ @@ -166,10 +168,10 @@ BITCOIN_QT_H = \ qt/walletview.h \ qt/winshutdownmonitor.h -RES_FONTS = \ +QT_RES_FONTS = \ qt/res/fonts/RobotoMono-Bold.ttf -RES_ICONS = \ +QT_RES_ICONS = \ qt/res/icons/add.png \ qt/res/icons/address-book.png \ qt/res/icons/bitcoin.ico \ @@ -227,6 +229,7 @@ BITCOIN_QT_BASE_CPP = \ qt/clientmodel.cpp \ qt/csvmodelwriter.cpp \ qt/guiutil.cpp \ + qt/initexecutor.cpp \ qt/intro.cpp \ qt/modaloverlay.cpp \ qt/networkstyle.cpp \ @@ -267,6 +270,7 @@ BITCOIN_QT_WALLET_CPP = \ qt/transactiondesc.cpp \ qt/transactiondescdialog.cpp \ qt/transactionfilterproxy.cpp \ + qt/transactionoverviewwidget.cpp \ qt/transactionrecord.cpp \ qt/transactiontablemodel.cpp \ qt/transactionview.cpp \ @@ -284,19 +288,19 @@ if ENABLE_WALLET BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_CPP) endif # ENABLE_WALLET -RES_ANIMATION = $(wildcard $(srcdir)/qt/res/animation/spinner-*.png) +QT_RES_ANIMATION = $(wildcard $(srcdir)/qt/res/animation/spinner-*.png) -BITCOIN_RC = qt/res/bitcoin-qt-res.rc +BITCOIN_QT_RC = qt/res/bitcoin-qt-res.rc BITCOIN_QT_INCLUDES = -DQT_NO_KEYWORDS -DQT_USE_QSTRINGBUILDER qt_libbitcoinqt_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ - $(QT_INCLUDES) $(QT_DBUS_INCLUDES) $(QR_CFLAGS) + $(QT_INCLUDES) $(QT_DBUS_INCLUDES) $(QR_CFLAGS) $(BOOST_CPPFLAGS) qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ - $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) + $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(QT_RES_FONTS) $(QT_RES_ICONS) $(QT_RES_ANIMATION) if TARGET_DARWIN qt_libbitcoinqt_a_SOURCES += $(BITCOIN_MM) endif @@ -318,32 +322,32 @@ bitcoin_qt_cxxflags = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) bitcoin_qt_sources = qt/main.cpp if TARGET_WINDOWS - bitcoin_qt_sources += $(BITCOIN_RC) + bitcoin_qt_sources += $(BITCOIN_QT_RC) endif -bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) +bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_NODE) if ENABLE_WALLET bitcoin_qt_ldadd += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif if ENABLE_ZMQ bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ - $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ +bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ + $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX qt_bitcoin_qt_CPPFLAGS = $(bitcoin_qt_cppflags) qt_bitcoin_qt_CXXFLAGS = $(bitcoin_qt_cxxflags) -qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) +qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) init/bitcoin-qt.cpp qt_bitcoin_qt_LDADD = $(bitcoin_qt_ldadd) qt_bitcoin_qt_LDFLAGS = $(bitcoin_qt_ldflags) qt_bitcoin_qt_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) bitcoin_gui_CPPFLAGS = $(bitcoin_qt_cppflags) bitcoin_gui_CXXFLAGS = $(bitcoin_qt_cxxflags) -bitcoin_gui_SOURCES = $(bitcoin_qt_sources) -bitcoin_gui_LDADD = $(bitcoin_qt_ldadd) +bitcoin_gui_SOURCES = $(bitcoin_qt_sources) init/bitcoin-gui.cpp +bitcoin_gui_LDADD = $(bitcoin_qt_ldadd) $(LIBBITCOIN_IPC) $(LIBBITCOIN_UTIL) $(LIBMULTIPROCESS_LIBS) bitcoin_gui_LDFLAGS = $(bitcoin_qt_ldflags) bitcoin_gui_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) @@ -354,13 +358,17 @@ SECONDARY: $(QT_QM) $(srcdir)/qt/bitcoinstrings.cpp: FORCE @test -n $(XGETTEXT) || echo "xgettext is required for updating translations" - $(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" $(PYTHON) ../share/qt/extract_strings_qt.py $(libbitcoin_server_a_SOURCES) $(libbitcoin_wallet_a_SOURCES) $(libbitcoin_common_a_SOURCES) $(libbitcoin_zmq_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) $(libbitcoin_util_a_SOURCES) + $(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" $(PYTHON) ../share/qt/extract_strings_qt.py $(libbitcoin_node_a_SOURCES) $(libbitcoin_wallet_a_SOURCES) $(libbitcoin_common_a_SOURCES) $(libbitcoin_zmq_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) $(libbitcoin_util_a_SOURCES) +# The resulted bitcoin_en.xlf source file should follow Transifex requirements. +# See: https://docs.transifex.com/formats/xliff#how-to-distinguish-between-a-source-file-and-a-translation-file translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCOIN_QT_BASE_CPP) qt/bitcoin.cpp $(BITCOIN_QT_WINDOWS_CPP) $(BITCOIN_QT_WALLET_CPP) $(BITCOIN_QT_H) $(BITCOIN_MM) @test -n $(LUPDATE) || echo "lupdate is required for updating translations" $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LUPDATE) -no-obsolete -I $(srcdir) -locations relative $^ -ts $(srcdir)/qt/locale/bitcoin_en.ts @test -n $(LCONVERT) || echo "lconvert is required for updating translations" - $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LCONVERT) -o $(srcdir)/qt/locale/bitcoin_en.xlf -i $(srcdir)/qt/locale/bitcoin_en.ts + $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LCONVERT) -drop-translations -o $(srcdir)/qt/locale/bitcoin_en.xlf -i $(srcdir)/qt/locale/bitcoin_en.ts + @$(SED) -i.old -e 's|source-language="en" target-language="en"|source-language="en"|' -e '/<\/target>/d' $(srcdir)/qt/locale/bitcoin_en.xlf + @rm -f $(srcdir)/qt/locale/bitcoin_en.xlf.old $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) @test -f $(RCC) @@ -368,7 +376,7 @@ $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin_locale --format-version 1 $(@D)/temp_$( $@ @rm $(@D)/temp_$( $@ @@ -388,11 +396,10 @@ QT_BASE_TLD = $(shell tar tf $(QT_BASE_PATH) --exclude='*/*') bitcoin_qt_apk: FORCE mkdir -p $(APK_LIB_DIR) - cp $(dir $(CC))../sysroot/usr/lib/$(host_alias)/libc++_shared.so $(APK_LIB_DIR) + cp $(dir $(lastword $(CC)))../sysroot/usr/lib/$(host_alias)/libc++_shared.so $(APK_LIB_DIR) tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/jar/src --strip-components=5 tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/java/src --strip-components=5 - tar xf $(QT_BASE_PATH) -C qt/android/res/ $(QT_BASE_TLD)src/android/java/res --strip-components=5 - cp qt/bitcoin-qt $(APK_LIB_DIR)/libbitcoin-qt.so + cp qt/bitcoin-qt $(APK_LIB_DIR)/libbitcoin-qt_$(ANDROID_ARCH).so cd qt/android && gradle wrapper --gradle-version=6.6.1 cd qt/android && ./gradlew build diff --git a/src/Makefile.qt_locale.include b/src/Makefile.qt_locale.include index aea42fd9023a..8638bd024cd4 100644 --- a/src/Makefile.qt_locale.include +++ b/src/Makefile.qt_locale.include @@ -1,7 +1,7 @@ QT_TS = \ - qt/locale/bitcoin_af.ts \ qt/locale/bitcoin_am.ts \ qt/locale/bitcoin_ar.ts \ + qt/locale/bitcoin_az.ts \ qt/locale/bitcoin_be.ts \ qt/locale/bitcoin_bg.ts \ qt/locale/bitcoin_bn.ts \ @@ -13,7 +13,6 @@ QT_TS = \ qt/locale/bitcoin_de.ts \ qt/locale/bitcoin_el.ts \ qt/locale/bitcoin_en.ts \ - qt/locale/bitcoin_en_GB.ts \ qt/locale/bitcoin_eo.ts \ qt/locale/bitcoin_es.ts \ qt/locale/bitcoin_es_CL.ts \ @@ -27,9 +26,12 @@ QT_TS = \ qt/locale/bitcoin_fi.ts \ qt/locale/bitcoin_fil.ts \ qt/locale/bitcoin_fr.ts \ + qt/locale/bitcoin_ga.ts \ + qt/locale/bitcoin_gd.ts \ + qt/locale/bitcoin_gl.ts \ qt/locale/bitcoin_gl_ES.ts \ + qt/locale/bitcoin_gu.ts \ qt/locale/bitcoin_he.ts \ - qt/locale/bitcoin_hi.ts \ qt/locale/bitcoin_hr.ts \ qt/locale/bitcoin_hu.ts \ qt/locale/bitcoin_id.ts \ @@ -38,6 +40,7 @@ QT_TS = \ qt/locale/bitcoin_ja.ts \ qt/locale/bitcoin_ka.ts \ qt/locale/bitcoin_kk.ts \ + qt/locale/bitcoin_kl.ts \ qt/locale/bitcoin_km.ts \ qt/locale/bitcoin_ko.ts \ qt/locale/bitcoin_ku_IQ.ts \ @@ -54,12 +57,14 @@ QT_TS = \ qt/locale/bitcoin_nb.ts \ qt/locale/bitcoin_ne.ts \ qt/locale/bitcoin_nl.ts \ + qt/locale/bitcoin_no.ts \ qt/locale/bitcoin_pam.ts \ qt/locale/bitcoin_pl.ts \ qt/locale/bitcoin_pt.ts \ qt/locale/bitcoin_pt_BR.ts \ qt/locale/bitcoin_ro.ts \ qt/locale/bitcoin_ru.ts \ + qt/locale/bitcoin_sc.ts \ qt/locale/bitcoin_si.ts \ qt/locale/bitcoin_sk.ts \ qt/locale/bitcoin_sl.ts \ @@ -68,11 +73,13 @@ QT_TS = \ qt/locale/bitcoin_sr.ts \ qt/locale/bitcoin_sr@latin.ts \ qt/locale/bitcoin_sv.ts \ + qt/locale/bitcoin_sw.ts \ qt/locale/bitcoin_szl.ts \ qt/locale/bitcoin_ta.ts \ qt/locale/bitcoin_te.ts \ qt/locale/bitcoin_th.ts \ qt/locale/bitcoin_tr.ts \ + qt/locale/bitcoin_ug.ts \ qt/locale/bitcoin_uk.ts \ qt/locale/bitcoin_ur.ts \ qt/locale/bitcoin_uz@Cyrl.ts \ diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 91a5e9fd9b44..89c659d4b921 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -7,6 +7,7 @@ TESTS += qt/test/test_bitcoin-qt TEST_QT_MOC_CPP = \ qt/test/moc_apptests.cpp \ + qt/test/moc_optiontests.cpp \ qt/test/moc_rpcnestedtests.cpp \ qt/test/moc_uritests.cpp @@ -19,16 +20,19 @@ endif # ENABLE_WALLET TEST_QT_H = \ qt/test/addressbooktests.h \ qt/test/apptests.h \ + qt/test/optiontests.h \ qt/test/rpcnestedtests.h \ qt/test/uritests.h \ qt/test/util.h \ qt/test/wallettests.h qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ - $(QT_INCLUDES) $(QT_TEST_INCLUDES) + $(QT_INCLUDES) $(QT_TEST_INCLUDES) $(BOOST_CPPFLAGS) qt_test_test_bitcoin_qt_SOURCES = \ + init/bitcoin-qt.cpp \ qt/test/apptests.cpp \ + qt/test/optiontests.cpp \ qt/test/rpcnestedtests.cpp \ qt/test/test_main.cpp \ qt/test/uritests.cpp \ @@ -43,7 +47,7 @@ endif # ENABLE_WALLET nodist_qt_test_test_bitcoin_qt_SOURCES = $(TEST_QT_MOC_CPP) -qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_SERVER) $(LIBTEST_UTIL) +qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_NODE) $(LIBTEST_UTIL) if ENABLE_WALLET qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif @@ -51,7 +55,7 @@ if ENABLE_ZMQ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ + $(LIBMEMENV) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index fc2fd8016603..74c30f1caf9c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -6,7 +6,7 @@ if ENABLE_FUZZ_BINARY noinst_PROGRAMS += test/fuzz/fuzz endif -if !ENABLE_FUZZ +if ENABLE_TESTS bin_PROGRAMS += test/test_bitcoin endif @@ -16,6 +16,7 @@ FUZZ_BINARY=test/fuzz/fuzz$(EXEEXT) JSON_TEST_FILES = \ test/data/script_tests.json \ + test/data/bip341_wallet_vectors.json \ test/data/base58_encode_decode.json \ test/data/blockfilters.json \ test/data/key_io_valid.json \ @@ -37,7 +38,7 @@ BITCOIN_TEST_SUITE = \ FUZZ_SUITE_LD_COMMON = \ $(LIBTEST_UTIL) \ $(LIBTEST_FUZZ) \ - $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ @@ -46,10 +47,9 @@ FUZZ_SUITE_LD_COMMON = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ - $(BOOST_LIBS) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ + $(MINISKETCH_LIBS) \ $(EVENT_LIBS) \ $(EVENT_PTHREADS_LIBS) @@ -63,11 +63,12 @@ endif # test_bitcoin binary # BITCOIN_TESTS =\ - test/arith_uint256_tests.cpp \ - test/scriptnum10.h \ test/addrman_tests.cpp \ - test/amount_tests.cpp \ test/allocator_tests.cpp \ + test/amount_tests.cpp \ + test/argsman_tests.cpp \ + test/arith_uint256_tests.cpp \ + test/banman_tests.cpp \ test/base32_tests.cpp \ test/base58_tests.cpp \ test/base64_tests.cpp \ @@ -75,8 +76,9 @@ BITCOIN_TESTS =\ test/bip32_tests.cpp \ test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ - test/blockfilter_tests.cpp \ test/blockfilter_index_tests.cpp \ + test/blockfilter_tests.cpp \ + test/blockmanager_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ @@ -86,27 +88,31 @@ BITCOIN_TESTS =\ test/compress_tests.cpp \ test/crypto_tests.cpp \ test/cuckoocache_tests.cpp \ + test/dbwrapper_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ + test/headers_sync_chainwork_tests.cpp \ + test/httpserver_tests.cpp \ test/i2p_tests.cpp \ test/interfaces_tests.cpp \ test/key_io_tests.cpp \ test/key_tests.cpp \ test/logging_tests.cpp \ - test/dbwrapper_tests.cpp \ - test/validation_tests.cpp \ test/mempool_tests.cpp \ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/miniscript_tests.cpp \ + test/minisketch_tests.cpp \ test/multisig_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ + test/orphanage_tests.cpp \ test/pmt_tests.cpp \ test/policy_fee_tests.cpp \ test/policyestimator_tests.cpp \ @@ -114,13 +120,19 @@ BITCOIN_TESTS =\ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ + test/rbf_tests.cpp \ + test/rest_tests.cpp \ + test/result_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_p2sh_tests.cpp \ - test/script_tests.cpp \ + test/script_parse_tests.cpp \ + test/script_segwit_tests.cpp \ test/script_standard_tests.cpp \ + test/script_tests.cpp \ + test/scriptnum10.h \ test/scriptnum_tests.cpp \ test/serfloat_tests.cpp \ test/serialize_tests.cpp \ @@ -132,33 +144,42 @@ BITCOIN_TESTS =\ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ - test/util_threadnames_tests.cpp \ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ test/txindex_tests.cpp \ + test/txpackage_tests.cpp \ + test/txreconciliation_tests.cpp \ test/txrequest_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/uint256_tests.cpp \ test/util_tests.cpp \ + test/util_threadnames_tests.cpp \ test/validation_block_tests.cpp \ test/validation_chainstate_tests.cpp \ test/validation_chainstatemanager_tests.cpp \ test/validation_flush_tests.cpp \ + test/validation_tests.cpp \ test/validationinterface_tests.cpp \ test/versionbits_tests.cpp if ENABLE_WALLET BITCOIN_TESTS += \ + wallet/test/feebumper_tests.cpp \ wallet/test/psbt_wallet_tests.cpp \ + wallet/test/spend_tests.cpp \ wallet/test/wallet_tests.cpp \ wallet/test/walletdb_tests.cpp \ wallet/test/wallet_crypto_tests.cpp \ + wallet/test/wallet_transaction_tests.cpp \ wallet/test/coinselector_tests.cpp \ + wallet/test/availablecoins_tests.cpp \ wallet/test/init_tests.cpp \ wallet/test/ismine_tests.cpp \ - wallet/test/scriptpubkeyman_tests.cpp + wallet/test/rpc_util_tests.cpp \ + wallet/test/scriptpubkeyman_tests.cpp \ + wallet/test/walletload_tests.cpp FUZZ_SUITE_LD_COMMON +=\ $(SQLITE_LIBS) \ @@ -168,23 +189,32 @@ if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif +FUZZ_WALLET_SRC = \ + wallet/test/fuzz/coinselection.cpp \ + wallet/test/fuzz/parse_iso8601.cpp + +if USE_SQLITE +FUZZ_WALLET_SRC += \ + wallet/test/fuzz/notifications.cpp +endif # USE_SQLITE BITCOIN_TEST_SUITE += \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ wallet/test/init_test_fixture.cpp \ wallet/test/init_test_fixture.h -endif +endif # ENABLE_WALLET test_test_bitcoin_SOURCES = $(BITCOIN_TEST_SUITE) $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) -test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(EVENT_CFLAGS) +test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(BOOST_CPPFLAGS) $(EVENT_CFLAGS) test_test_bitcoin_LDADD = $(LIBTEST_UTIL) if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) +test_test_bitcoin_CPPFLAGS += $(BDB_CPPFLAGS) endif -test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ - $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) +test_test_bitcoin_LDADD += $(LIBBITCOIN_NODE) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ + $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) @@ -195,16 +225,14 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -FUZZ_SUITE_LDFLAGS_COMMON = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) - if ENABLE_FUZZ_BINARY -test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fuzz_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) +test_fuzz_fuzz_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) $(RUNTIME_LDFLAGS) test_fuzz_fuzz_SOURCES = \ + $(FUZZ_WALLET_SRC) \ test/fuzz/addition_overflow.cpp \ - test/fuzz/addrdb.cpp \ test/fuzz/addrman.cpp \ test/fuzz/asmap.cpp \ test/fuzz/asmap_direct.cpp \ @@ -212,6 +240,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/banman.cpp \ test/fuzz/base_encode_decode.cpp \ test/fuzz/bech32.cpp \ + test/fuzz/bitdeque.cpp \ test/fuzz/block.cpp \ test/fuzz/block_header.cpp \ test/fuzz/blockfilter.cpp \ @@ -227,10 +256,10 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/crypto_chacha20.cpp \ test/fuzz/crypto_chacha20_poly1305_aead.cpp \ test/fuzz/crypto_common.cpp \ + test/fuzz/crypto_diff_fuzz_chacha20.cpp \ test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ test/fuzz/crypto_poly1305.cpp \ test/fuzz/cuckoocache.cpp \ - test/fuzz/data_stream.cpp \ test/fuzz/decode_tx.cpp \ test/fuzz/descriptor_parse.cpp \ test/fuzz/deserialize.cpp \ @@ -251,6 +280,8 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/locale.cpp \ test/fuzz/merkleblock.cpp \ test/fuzz/message.cpp \ + test/fuzz/miniscript.cpp \ + test/fuzz/minisketch.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ test/fuzz/net.cpp \ @@ -260,7 +291,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/node_eviction.cpp \ test/fuzz/p2p_transport_serialization.cpp \ test/fuzz/parse_hd_keypath.cpp \ - test/fuzz/parse_iso8601.cpp \ test/fuzz/parse_numbers.cpp \ test/fuzz/parse_script.cpp \ test/fuzz/parse_univalue.cpp \ @@ -282,6 +312,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/script_bitcoin_consensus.cpp \ test/fuzz/script_descriptor_cache.cpp \ test/fuzz/script_flags.cpp \ + test/fuzz/script_format.cpp \ test/fuzz/script_interpreter.cpp \ test/fuzz/script_ops.cpp \ test/fuzz/script_sigcache.cpp \ @@ -303,6 +334,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/tx_in.cpp \ test/fuzz/tx_out.cpp \ test/fuzz/tx_pool.cpp \ + test/fuzz/txorphan.cpp \ test/fuzz/txrequest.cpp \ test/fuzz/utxo_snapshot.cpp \ test/fuzz/validation_load_mempool.cpp \ @@ -313,7 +345,7 @@ nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) $(BITCOIN_TESTS): $(GENERATED_TEST_FILES) -CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno test/fuzz/*.gcda test/fuzz/*.gcno test/util/*.gcda test/util/*.gcno $(GENERATED_TEST_FILES) $(BITCOIN_TESTS:=.log) +CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno test/fuzz/*.gcda test/fuzz/*.gcno test/util/*.gcda test/util/*.gcno $(GENERATED_TEST_FILES) $(addsuffix .log,$(basename $(BITCOIN_TESTS))) CLEANFILES += $(CLEAN_BITCOIN_TEST) @@ -335,26 +367,50 @@ bitcoin_test_clean : FORCE check-local: $(BITCOIN_TESTS:.cpp=.cpp.test) if BUILD_BITCOIN_TX - @echo "Running test/util/bitcoin-util-test.py..." - $(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py + @echo "Running test/util/test_runner.py..." + $(PYTHON) $(top_builddir)/test/util/test_runner.py endif @echo "Running test/util/rpcauth-test.py..." $(PYTHON) $(top_builddir)/test/util/rpcauth-test.py if TARGET_WINDOWS else if ENABLE_BENCH - @echo "Running bench/bench_bitcoin ..." - $(BENCH_BINARY) > /dev/null + @echo "Running bench/bench_bitcoin (one iteration sanity check, only high priority)..." + $(BENCH_BINARY) -sanity-check -priority-level=high > /dev/null endif endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check -if EMBEDDED_UNIVALUE - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check + +if ENABLE_TESTS +UNIVALUE_TESTS = univalue/test/object univalue/test/unitester +noinst_PROGRAMS += $(UNIVALUE_TESTS) +TESTS += $(UNIVALUE_TESTS) + +univalue_test_unitester_SOURCES = $(UNIVALUE_TEST_UNITESTER_INT) +univalue_test_unitester_LDADD = $(LIBUNIVALUE) +univalue_test_unitester_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" +univalue_test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +univalue_test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) +univalue_test_object_LDADD = $(LIBUNIVALUE) +univalue_test_object_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +univalue_test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) endif %.cpp.test: %.cpp - @echo Running tests: `cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1` from $< - $(AM_V_at)$(TEST_BINARY) --catch_system_errors=no -l test_suite -t "`cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1`" -- DEBUG_LOG_OUT > $<.log 2>&1 || (cat $<.log && false) + @echo Running tests: $$(\ + cat $< | \ + grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | \ + cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1\ + ) from $< + $(AM_V_at)export TEST_LOGFILE=$(abs_builddir)/$$(\ + echo $< | grep -E -o "(wallet/test/.*\.cpp|test/.*\.cpp)" | $(SED) -e s/\.cpp/.log/ \ + ) && \ + $(TEST_BINARY) --catch_system_errors=no -l test_suite -t "$$(\ + cat $< | \ + grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | \ + cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1\ + )" -- DEBUG_LOG_OUT > "$$TEST_LOGFILE" 2>&1 || (cat "$$TEST_LOGFILE" && false) %.json.h: %.json @$(MKDIR_P) $(@D) @@ -365,12 +421,3 @@ endif echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" - -%.raw.h: %.raw - @$(MKDIR_P) $(@D) - @{ \ - echo "static unsigned const char $(*F)_raw[] = {" && \ - $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ - echo "};"; \ - } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 2d772f2fca16..aa9c0527509b 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -10,16 +10,15 @@ EXTRA_LIBRARIES += \ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ - test/fuzz/util.h + test/fuzz/util.h \ + test/fuzz/util/mempool.h \ + test/fuzz/util/net.h -libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ test/fuzz/util.cpp \ + test/fuzz/util/mempool.cpp \ + test/fuzz/util/net.cpp \ $(TEST_FUZZ_H) - -LIBTEST_FUZZ += $(LIBBITCOIN_SERVER) -LIBTEST_FUZZ += $(LIBBITCOIN_COMMON) -LIBTEST_FUZZ += $(LIBBITCOIN_UTIL) -LIBTEST_FUZZ += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 85e50ebf7058..a4e8b3f842f8 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -5,21 +5,26 @@ LIBTEST_UTIL=libtest_util.a EXTRA_LIBRARIES += \ - $(LIBTEST_UTIL) + $(LIBTEST_UTIL) TEST_UTIL_H = \ - test/util/blockfilter.h \ - test/util/logging.h \ - test/util/mining.h \ - test/util/net.h \ - test/util/script.h \ - test/util/setup_common.h \ - test/util/str.h \ - test/util/transaction_utils.h \ - test/util/validation.h \ - test/util/wallet.h + test/util/blockfilter.h \ + test/util/chainstate.h \ + test/util/logging.h \ + test/util/mining.h \ + test/util/net.h \ + test/util/script.h \ + test/util/setup_common.h \ + test/util/str.h \ + test/util/transaction_utils.h \ + test/util/txmempool.h \ + test/util/validation.h -libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +if ENABLE_WALLET +TEST_UTIL_H += wallet/test/util.h +endif # ENABLE_WALLET + +libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_util_a_SOURCES = \ test/util/blockfilter.cpp \ @@ -30,11 +35,11 @@ libtest_util_a_SOURCES = \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ - test/util/validation.cpp \ - test/util/wallet.cpp \ - $(TEST_UTIL_H) + test/util/txmempool.cpp \ + test/util/validation.cpp + +if ENABLE_WALLET +libtest_util_a_SOURCES += wallet/test/util.cpp +endif # ENABLE_WALLET -LIBTEST_UTIL += $(LIBBITCOIN_SERVER) -LIBTEST_UTIL += $(LIBBITCOIN_COMMON) -LIBTEST_UTIL += $(LIBBITCOIN_UTIL) -LIBTEST_UTIL += $(LIBBITCOIN_CRYPTO_BASE) +libtest_util_a_SOURCES += $(TEST_UTIL_H) diff --git a/src/Makefile.univalue.include b/src/Makefile.univalue.include new file mode 100644 index 000000000000..3644e363688d --- /dev/null +++ b/src/Makefile.univalue.include @@ -0,0 +1,6 @@ +include univalue/sources.mk + +LIBUNIVALUE = libunivalue.la +noinst_LTLIBRARIES += $(LIBUNIVALUE) +libunivalue_la_SOURCES = $(UNIVALUE_LIB_SOURCES_INT) $(UNIVALUE_DIST_HEADERS_INT) $(UNIVALUE_LIB_HEADERS_INT) $(UNIVALUE_TEST_FILES_INT) +libunivalue_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index b8fd019bab9c..7106d819b0f3 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,73 +9,25 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include #include #include - -CBanEntry::CBanEntry(const UniValue& json) - : nVersion(json["version"].get_int()), nCreateTime(json["ban_created"].get_int64()), - nBanUntil(json["banned_until"].get_int64()) -{ -} - -UniValue CBanEntry::ToJson() const -{ - UniValue json(UniValue::VOBJ); - json.pushKV("version", nVersion); - json.pushKV("ban_created", nCreateTime); - json.pushKV("banned_until", nBanUntil); - return json; -} +#include namespace { -static const char* BANMAN_JSON_ADDR_KEY = "address"; - -/** - * Convert a `banmap_t` object to a JSON array. - * @param[in] bans Bans list to convert. - * @return a JSON array, similar to the one returned by the `listbanned` RPC. Suitable for - * passing to `BanMapFromJson()`. - */ -UniValue BanMapToJson(const banmap_t& bans) +class DbNotFoundError : public std::exception { - UniValue bans_json(UniValue::VARR); - for (const auto& it : bans) { - const auto& address = it.first; - const auto& ban_entry = it.second; - UniValue j = ban_entry.ToJson(); - j.pushKV(BANMAN_JSON_ADDR_KEY, address.ToString()); - bans_json.push_back(j); - } - return bans_json; -} - -/** - * Convert a JSON array to a `banmap_t` object. - * @param[in] bans_json JSON to convert, must be as returned by `BanMapToJson()`. - * @param[out] bans Bans list to create from the JSON. - * @throws std::runtime_error if the JSON does not have the expected fields or they contain - * unparsable values. - */ -void BanMapFromJson(const UniValue& bans_json, banmap_t& bans) -{ - for (const auto& ban_entry_json : bans_json.getValues()) { - CSubNet subnet; - const auto& subnet_str = ban_entry_json[BANMAN_JSON_ADDR_KEY].get_str(); - if (!LookupSubNet(subnet_str, subnet)) { - throw std::runtime_error( - strprintf("Cannot parse banned address or subnet: %s", subnet_str)); - } - bans.insert_or_assign(subnet, CBanEntry{ban_entry_json}); - } -} + using std::exception::exception; +}; template bool SerializeDB(Stream& stream, const Data& data) @@ -97,18 +49,17 @@ template bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data, int version) { // Generate random temporary filename - uint16_t randv = 0; - GetRandBytes((unsigned char*)&randv, sizeof(randv)); + const uint16_t randv{GetRand()}; std::string tmpfn = strprintf("%s.%04x", prefix, randv); // open temp output file, and associate with CAutoFile - fs::path pathTmp = gArgs.GetDataDirNet() / tmpfn; + fs::path pathTmp = gArgs.GetDataDirNet() / fs::u8path(tmpfn); FILE *file = fsbridge::fopen(pathTmp, "wb"); CAutoFile fileout(file, SER_DISK, version); if (fileout.IsNull()) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to open file %s", __func__, pathTmp.string()); + return error("%s: Failed to open file %s", __func__, fs::PathToString(pathTmp)); } // Serialize @@ -120,7 +71,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data if (!FileCommit(fileout.Get())) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to flush file %s", __func__, pathTmp.string()); + return error("%s: Failed to flush file %s", __func__, fs::PathToString(pathTmp)); } fileout.fclose(); @@ -134,53 +85,46 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data } template -bool DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) -{ - try { - CHashVerifier verifier(&stream); - // de-serialize file header (network specific magic number) and .. - unsigned char pchMsgTmp[4]; - verifier >> pchMsgTmp; - // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) - return error("%s: Invalid network magic number", __func__); - - // de-serialize data - verifier >> data; - - // verify checksum - if (fCheckSum) { - uint256 hashTmp; - stream >> hashTmp; - if (hashTmp != verifier.GetHash()) { - return error("%s: Checksum mismatch, data corrupted", __func__); - } +void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) +{ + CHashVerifier verifier(&stream); + // de-serialize file header (network specific magic number) and .. + unsigned char pchMsgTmp[4]; + verifier >> pchMsgTmp; + // ... verify the network matches ours + if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) { + throw std::runtime_error{"Invalid network magic number"}; + } + + // de-serialize data + verifier >> data; + + // verify checksum + if (fCheckSum) { + uint256 hashTmp; + stream >> hashTmp; + if (hashTmp != verifier.GetHash()) { + throw std::runtime_error{"Checksum mismatch, data corrupted"}; } } - catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - - return true; } template -bool DeserializeFileDB(const fs::path& path, Data& data, int version) +void DeserializeFileDB(const fs::path& path, Data& data, int version) { // open input file, and associate with CAutoFile FILE* file = fsbridge::fopen(path, "rb"); CAutoFile filein(file, SER_DISK, version); if (filein.IsNull()) { - LogPrintf("Missing or invalid file %s\n", path.string()); - return false; + throw DbNotFoundError{}; } - return DeserializeDB(filein, data); + DeserializeDB(filein, data); } } // namespace CBanDB::CBanDB(fs::path ban_list_path) - : m_banlist_dat(ban_list_path.string() + ".dat"), - m_banlist_json(ban_list_path.string() + ".json") + : m_banlist_dat(ban_list_path + ".dat"), + m_banlist_json(ban_list_path + ".json") { } @@ -197,23 +141,22 @@ bool CBanDB::Write(const banmap_t& banSet) return false; } -bool CBanDB::Read(banmap_t& banSet, bool& dirty) +bool CBanDB::Read(banmap_t& banSet) { - // If the JSON banlist does not exist, then try to read the non-upgraded banlist.dat. + if (fs::exists(m_banlist_dat)) { + LogPrintf("banlist.dat ignored because it can only be read by " PACKAGE_NAME " version 22.x. Remove %s to silence this warning.\n", fs::quoted(fs::PathToString(m_banlist_dat))); + } + // If the JSON banlist does not exist, then recreate it if (!fs::exists(m_banlist_json)) { - // If this succeeds then we need to flush to disk in order to create the JSON banlist. - dirty = true; - return DeserializeFileDB(m_banlist_dat, banSet, CLIENT_VERSION); + return false; } - dirty = false; - std::map settings; std::vector errors; if (!util::ReadSettings(m_banlist_json, settings, errors)) { for (const auto& err : errors) { - LogPrintf("Cannot load banlist %s: %s\n", m_banlist_json.string(), err); + LogPrintf("Cannot load banlist %s: %s\n", fs::PathToString(m_banlist_json), err); } return false; } @@ -221,36 +164,54 @@ bool CBanDB::Read(banmap_t& banSet, bool& dirty) try { BanMapFromJson(settings[JSON_KEY], banSet); } catch (const std::runtime_error& e) { - LogPrintf("Cannot parse banlist %s: %s\n", m_banlist_json.string(), e.what()); + LogPrintf("Cannot parse banlist %s: %s\n", fs::PathToString(m_banlist_json), e.what()); return false; } return true; } -CAddrDB::CAddrDB() -{ - pathAddr = gArgs.GetDataDirNet() / "peers.dat"; -} - -bool CAddrDB::Write(const CAddrMan& addr) +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr) { + const auto pathAddr = args.GetDataDirNet() / "peers.dat"; return SerializeFileDB("peers", pathAddr, addr, CLIENT_VERSION); } -bool CAddrDB::Read(CAddrMan& addr) +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) { - return DeserializeFileDB(pathAddr, addr, CLIENT_VERSION); + DeserializeDB(ssPeers, addr, false); } -bool CAddrDB::Read(CAddrMan& addr, CDataStream& ssPeers) +std::optional LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr& addrman) { - bool ret = DeserializeDB(ssPeers, addr, false); - if (!ret) { - // Ensure addrman is left in a clean state - addr.Clear(); + auto check_addrman = std::clamp(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); + addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + + const auto start{SteadyClock::now()}; + const auto path_addr{args.GetDataDirNet() / "peers.dat"}; + try { + DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION); + LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), Ticks(SteadyClock::now() - start)); + } catch (const DbNotFoundError&) { + // Addrman can be in an inconsistent state after failure, reset it + addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); + DumpPeerAddresses(args, *addrman); + } catch (const InvalidAddrManVersionError&) { + if (!RenameOver(path_addr, (fs::path)path_addr + ".bak")) { + addrman = nullptr; + return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); + } + // Addrman can be in an inconsistent state after failure, reset it + addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); + DumpPeerAddresses(args, *addrman); + } catch (const std::exception& e) { + addrman = nullptr; + return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), + e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr))); } - return ret; + return std::nullopt; } void DumpAnchors(const fs::path& anchors_db_path, const std::vector& anchors) @@ -262,9 +223,10 @@ void DumpAnchors(const fs::path& anchors_db_path, const std::vector& a std::vector ReadAnchors(const fs::path& anchors_db_path) { std::vector anchors; - if (DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT)) { - LogPrintf("Loaded %i addresses from %s\n", anchors.size(), anchors_db_path.filename()); - } else { + try { + DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT); + LogPrintf("Loaded %i addresses from %s\n", anchors.size(), fs::quoted(fs::PathToString(anchors_db_path.filename()))); + } catch (const std::exception&) { anchors.clear(); } diff --git a/src/addrdb.h b/src/addrdb.h index 399103c99199..627ef3ac3ce2 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,75 +8,23 @@ #include #include // For banmap_t -#include #include -#include +#include #include +class ArgsManager; +class AddrMan; class CAddress; -class CAddrMan; class CDataStream; +class NetGroupManager; +struct bilingual_str; -class CBanEntry -{ -public: - static const int CURRENT_VERSION=1; - int nVersion; - int64_t nCreateTime; - int64_t nBanUntil; - - CBanEntry() - { - SetNull(); - } - - explicit CBanEntry(int64_t nCreateTimeIn) - { - SetNull(); - nCreateTime = nCreateTimeIn; - } - - /** - * Create a ban entry from JSON. - * @param[in] json A JSON representation of a ban entry, as created by `ToJson()`. - * @throw std::runtime_error if the JSON does not have the expected fields. - */ - explicit CBanEntry(const UniValue& json); - - SERIALIZE_METHODS(CBanEntry, obj) - { - uint8_t ban_reason = 2; //! For backward compatibility - READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, ban_reason); - } +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); +/** Only used by tests. */ +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers); - void SetNull() - { - nVersion = CBanEntry::CURRENT_VERSION; - nCreateTime = 0; - nBanUntil = 0; - } - - /** - * Generate a JSON representation of this ban entry. - * @return JSON suitable for passing to the `CBanEntry(const UniValue&)` constructor. - */ - UniValue ToJson() const; -}; - -/** Access to the (IP) address database (peers.dat) */ -class CAddrDB -{ -private: - fs::path pathAddr; -public: - CAddrDB(); - bool Write(const CAddrMan& addr); - bool Read(CAddrMan& addr); - static bool Read(CAddrMan& addr, CDataStream& ssPeers); -}; - -/** Access to the banlist databases (banlist.json and banlist.dat) */ +/** Access to the banlist database (banlist.json) */ class CBanDB { private: @@ -95,13 +43,14 @@ class CBanDB * Read the banlist from disk. * @param[out] banSet The loaded list. Set if `true` is returned, otherwise it is left * in an undefined state. - * @param[out] dirty Indicates whether the loaded list needs flushing to disk. Set if - * `true` is returned, otherwise it is left in an undefined state. * @return true on success */ - bool Read(banmap_t& banSet, bool& dirty); + bool Read(banmap_t& banSet); }; +/** Returns an error string on failure */ +std::optional LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr& addrman); + /** * Dump the anchor IP address database (anchors.dat) * diff --git a/src/addrman.cpp b/src/addrman.cpp index 8192b4eba666..f16ff2230b93 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,75 +1,102 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2020 The Bitcoin Core developers +// Copyright (c) 2012-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include +#include #include +#include +#include #include +#include +#include +#include +#include +#include #include #include -#include -#include -int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector &asmap) const +/** Over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread */ +static constexpr uint32_t ADDRMAN_TRIED_BUCKETS_PER_GROUP{8}; +/** Over how many buckets entries with new addresses originating from a single group are spread */ +static constexpr uint32_t ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP{64}; +/** Maximum number of times an address can occur in the new table */ +static constexpr int32_t ADDRMAN_NEW_BUCKETS_PER_ADDRESS{8}; +/** How old addresses can maximally be */ +static constexpr auto ADDRMAN_HORIZON{30 * 24h}; +/** After how many failed attempts we give up on a new node */ +static constexpr int32_t ADDRMAN_RETRIES{3}; +/** How many successive failures are allowed ... */ +static constexpr int32_t ADDRMAN_MAX_FAILURES{10}; +/** ... in at least this duration */ +static constexpr auto ADDRMAN_MIN_FAIL{7 * 24h}; +/** How recent a successful connection should be before we allow an address to be evicted from tried */ +static constexpr auto ADDRMAN_REPLACEMENT{4h}; +/** The maximum number of tried addr collisions to store */ +static constexpr size_t ADDRMAN_SET_TRIED_COLLISION_SIZE{10}; +/** The maximum time we'll spend trying to resolve a tried table collision */ +static constexpr auto ADDRMAN_TEST_WINDOW{40min}; + +int AddrInfo::GetTriedBucket(const uint256& nKey, const NetGroupManager& netgroupman) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); - int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i\n", ToStringIP(), mapped_as, tried_bucket); - return tried_bucket; + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << netgroupman.GetGroup(*this) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); + return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector &asmap) const +int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const NetGroupManager& netgroupman) const { - std::vector vchSourceGroupKey = src.GetGroup(asmap); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); + std::vector vchSourceGroupKey = netgroupman.GetGroup(src); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << netgroupman.GetGroup(*this) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); - int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i\n", ToStringIP(), mapped_as, new_bucket); - return new_bucket; + return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } -int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const +int AddrInfo::GetBucketPosition(const uint256& nKey, bool fNew, int nBucket) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << (fNew ? uint8_t{'N'} : uint8_t{'K'}) << nBucket << GetKey()).GetCheapHash(); return hash1 % ADDRMAN_BUCKET_SIZE; } -bool CAddrInfo::IsTerrible(int64_t nNow) const +bool AddrInfo::IsTerrible(NodeSeconds now) const { - if (nLastTry && nLastTry >= nNow - 60) // never remove things tried in the last minute + if (now - m_last_try <= 1min) { // never remove things tried in the last minute return false; + } - if (nTime > nNow + 10 * 60) // came in a flying DeLorean + if (nTime > now + 10min) { // came in a flying DeLorean return true; + } - if (nTime == 0 || nNow - nTime > ADDRMAN_HORIZON_DAYS * 24 * 60 * 60) // not seen in recent history + if (now - nTime > ADDRMAN_HORIZON) { // not seen in recent history return true; + } - if (nLastSuccess == 0 && nAttempts >= ADDRMAN_RETRIES) // tried N times and never a success + if (TicksSinceEpoch(m_last_success) == 0 && nAttempts >= ADDRMAN_RETRIES) { // tried N times and never a success return true; + } - if (nNow - nLastSuccess > ADDRMAN_MIN_FAIL_DAYS * 24 * 60 * 60 && nAttempts >= ADDRMAN_MAX_FAILURES) // N successive failures in the last week + if (now - m_last_success > ADDRMAN_MIN_FAIL && nAttempts >= ADDRMAN_MAX_FAILURES) { // N successive failures in the last week return true; + } return false; } -double CAddrInfo::GetChance(int64_t nNow) const +double AddrInfo::GetChance(NodeSeconds now) const { double fChance = 1.0; - int64_t nSinceLastTry = std::max(nNow - nLastTry, 0); // deprioritize very recent attempts away - if (nSinceLastTry < 60 * 10) + if (now - m_last_try < 10min) { fChance *= 0.01; + } // deprioritize 66% after each failed attempt, but at most 1/28th to avoid the search taking forever or overly penalizing outages. fChance *= pow(0.66, std::min(nAttempts, 8)); @@ -77,39 +104,304 @@ double CAddrInfo::GetChance(int64_t nNow) const return fChance; } -void CAddrMan::RemoveInvalid() +AddrManImpl::AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio) + : insecure_rand{deterministic} + , nKey{deterministic ? uint256{1} : insecure_rand.rand256()} + , m_consistency_check_ratio{consistency_check_ratio} + , m_netgroupman{netgroupman} +{ + for (auto& bucket : vvNew) { + for (auto& entry : bucket) { + entry = -1; + } + } + for (auto& bucket : vvTried) { + for (auto& entry : bucket) { + entry = -1; + } + } +} + +AddrManImpl::~AddrManImpl() +{ + nKey.SetNull(); +} + +template +void AddrManImpl::Serialize(Stream& s_) const { - for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { - for (size_t i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { - const auto id = vvNew[bucket][i]; - if (id != -1 && !mapInfo[id].IsValid()) { - ClearNew(bucket, i); + LOCK(cs); + + /** + * Serialized format. + * * format version byte (@see `Format`) + * * lowest compatible format version byte. This is used to help old software decide + * whether to parse the file. For example: + * * Bitcoin Core version N knows how to parse up to format=3. If a new format=4 is + * introduced in version N+1 that is compatible with format=3 and it is known that + * version N will be able to parse it, then version N+1 will write + * (format=4, lowest_compatible=3) in the first two bytes of the file, and so + * version N will still try to parse it. + * * Bitcoin Core version N+2 introduces a new incompatible format=5. It will write + * (format=5, lowest_compatible=5) and so any versions that do not know how to parse + * format=5 will not try to read the file. + * * nKey + * * nNew + * * nTried + * * number of "new" buckets XOR 2**30 + * * all new addresses (total count: nNew) + * * all tried addresses (total count: nTried) + * * for each new bucket: + * * number of elements + * * for each element: index in the serialized "all new addresses" + * * asmap checksum + * + * 2**30 is xorred with the number of buckets to make addrman deserializer v0 detect it + * as incompatible. This is necessary because it did not check the version number on + * deserialization. + * + * vvNew, vvTried, mapInfo, mapAddr and vRandom are never encoded explicitly; + * they are instead reconstructed from the other information. + * + * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports + * changes to the ADDRMAN_ parameters without breaking the on-disk structure. + * + * We don't use SERIALIZE_METHODS since the serialization and deserialization code has + * very little in common. + */ + + // Always serialize in the latest version (FILE_FORMAT). + + OverrideStream s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT); + + s << static_cast(FILE_FORMAT); + + // Increment `lowest_compatible` iff a newly introduced format is incompatible with + // the previous one. + static constexpr uint8_t lowest_compatible = Format::V4_MULTIPORT; + s << static_cast(INCOMPATIBILITY_BASE + lowest_compatible); + + s << nKey; + s << nNew; + s << nTried; + + int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); + s << nUBuckets; + std::unordered_map mapUnkIds; + int nIds = 0; + for (const auto& entry : mapInfo) { + mapUnkIds[entry.first] = nIds; + const AddrInfo& info = entry.second; + if (info.nRefCount) { + assert(nIds != nNew); // this means nNew was wrong, oh ow + s << info; + nIds++; + } + } + nIds = 0; + for (const auto& entry : mapInfo) { + const AddrInfo& info = entry.second; + if (info.fInTried) { + assert(nIds != nTried); // this means nTried was wrong, oh ow + s << info; + nIds++; + } + } + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { + int nSize = 0; + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[bucket][i] != -1) + nSize++; + } + s << nSize; + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[bucket][i] != -1) { + int nIndex = mapUnkIds[vvNew[bucket][i]]; + s << nIndex; } } } + // Store asmap checksum after bucket entries so that it + // can be ignored by older clients for backward compatibility. + s << m_netgroupman.GetAsmapChecksum(); +} + +template +void AddrManImpl::Unserialize(Stream& s_) +{ + LOCK(cs); + + assert(vRandom.empty()); + + Format format; + s_ >> Using>(format); + + int stream_version = s_.GetVersion(); + if (format >= Format::V3_BIP155) { + // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress + // unserialize methods know that an address in addrv2 format is coming. + stream_version |= ADDRV2_FORMAT; + } + + OverrideStream s(&s_, s_.GetType(), stream_version); + + uint8_t compat; + s >> compat; + if (compat < INCOMPATIBILITY_BASE) { + throw std::ios_base::failure(strprintf( + "Corrupted addrman database: The compat value (%u) " + "is lower than the expected minimum value %u.", + compat, INCOMPATIBILITY_BASE)); + } + const uint8_t lowest_compatible = compat - INCOMPATIBILITY_BASE; + if (lowest_compatible > FILE_FORMAT) { + throw InvalidAddrManVersionError(strprintf( + "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " + "but the maximum supported by this version of %s is %u.", + uint8_t{format}, lowest_compatible, PACKAGE_NAME, uint8_t{FILE_FORMAT})); + } + + s >> nKey; + s >> nNew; + s >> nTried; + int nUBuckets = 0; + s >> nUBuckets; + if (format >= Format::V1_DETERMINISTIC) { + nUBuckets ^= (1 << 30); + } + + if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nNew < 0) { + throw std::ios_base::failure( + strprintf("Corrupt AddrMan serialization: nNew=%d, should be in [0, %d]", + nNew, + ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); + } - for (size_t bucket = 0; bucket < ADDRMAN_TRIED_BUCKET_COUNT; ++bucket) { - for (size_t i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { - const auto id = vvTried[bucket][i]; - if (id == -1) { - continue; + if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nTried < 0) { + throw std::ios_base::failure( + strprintf("Corrupt AddrMan serialization: nTried=%d, should be in [0, %d]", + nTried, + ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); + } + + // Deserialize entries from the new table. + for (int n = 0; n < nNew; n++) { + AddrInfo& info = mapInfo[n]; + s >> info; + mapAddr[info] = n; + info.nRandomPos = vRandom.size(); + vRandom.push_back(n); + } + nIdCount = nNew; + + // Deserialize entries from the tried table. + int nLost = 0; + for (int n = 0; n < nTried; n++) { + AddrInfo info; + s >> info; + int nKBucket = info.GetTriedBucket(nKey, m_netgroupman); + int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); + if (info.IsValid() + && vvTried[nKBucket][nKBucketPos] == -1) { + info.nRandomPos = vRandom.size(); + info.fInTried = true; + vRandom.push_back(nIdCount); + mapInfo[nIdCount] = info; + mapAddr[info] = nIdCount; + vvTried[nKBucket][nKBucketPos] = nIdCount; + nIdCount++; + } else { + nLost++; + } + } + nTried -= nLost; + + // Store positions in the new table buckets to apply later (if possible). + // An entry may appear in up to ADDRMAN_NEW_BUCKETS_PER_ADDRESS buckets, + // so we store all bucket-entry_index pairs to iterate through later. + std::vector> bucket_entries; + + for (int bucket = 0; bucket < nUBuckets; ++bucket) { + int num_entries{0}; + s >> num_entries; + for (int n = 0; n < num_entries; ++n) { + int entry_index{0}; + s >> entry_index; + if (entry_index >= 0 && entry_index < nNew) { + bucket_entries.emplace_back(bucket, entry_index); } - const auto& addr_info = mapInfo[id]; - if (addr_info.IsValid()) { - continue; + } + } + + // If the bucket count and asmap checksum haven't changed, then attempt + // to restore the entries to the buckets/positions they were in before + // serialization. + uint256 supplied_asmap_checksum{m_netgroupman.GetAsmapChecksum()}; + uint256 serialized_asmap_checksum; + if (format >= Format::V2_ASMAP) { + s >> serialized_asmap_checksum; + } + const bool restore_bucketing{nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && + serialized_asmap_checksum == supplied_asmap_checksum}; + + if (!restore_bucketing) { + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + } + + for (auto bucket_entry : bucket_entries) { + int bucket{bucket_entry.first}; + const int entry_index{bucket_entry.second}; + AddrInfo& info = mapInfo[entry_index]; + + // Don't store the entry in the new bucket if it's not a valid address for our addrman + if (!info.IsValid()) continue; + + // The entry shouldn't appear in more than + // ADDRMAN_NEW_BUCKETS_PER_ADDRESS. If it has already, just skip + // this bucket_entry. + if (info.nRefCount >= ADDRMAN_NEW_BUCKETS_PER_ADDRESS) continue; + + int bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (restore_bucketing && vvNew[bucket][bucket_position] == -1) { + // Bucketing has not changed, using existing bucket positions for the new table + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; + } else { + // In case the new table data cannot be used (bucket count wrong or new asmap), + // try to give them a reference based on their primary source address. + bucket = info.GetNewBucket(nKey, m_netgroupman); + bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][bucket_position] == -1) { + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; } - vvTried[bucket][i] = -1; - --nTried; - SwapRandom(addr_info.nRandomPos, vRandom.size() - 1); - vRandom.pop_back(); - mapAddr.erase(addr_info); - mapInfo.erase(id); - m_tried_collisions.erase(id); } } + + // Prune new entries with refcount 0 (as a result of collisions or invalid address). + int nLostUnk = 0; + for (auto it = mapInfo.cbegin(); it != mapInfo.cend(); ) { + if (it->second.fInTried == false && it->second.nRefCount == 0) { + const auto itCopy = it++; + Delete(itCopy->first); + ++nLostUnk; + } else { + ++it; + } + } + if (nLost + nLostUnk > 0) { + LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions or invalid addresses\n", nLostUnk, nLost); + } + + const int check_code{CheckAddrman()}; + if (check_code != 0) { + throw std::ios_base::failure(strprintf( + "Corrupt data. Consistency check failed with code %s", + check_code)); + } } -CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) +AddrInfo* AddrManImpl::Find(const CService& addr, int* pnId) { AssertLockHeld(cs); @@ -124,12 +416,12 @@ CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) return nullptr; } -CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) +AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) { AssertLockHeld(cs); int nId = nIdCount++; - mapInfo[nId] = CAddrInfo(addr, addrSource); + mapInfo[nId] = AddrInfo(addr, addrSource); mapAddr[addr] = nId; mapInfo[nId].nRandomPos = vRandom.size(); vRandom.push_back(nId); @@ -138,7 +430,7 @@ CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, in return &mapInfo[nId]; } -void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) +void AddrManImpl::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const { AssertLockHeld(cs); @@ -150,22 +442,24 @@ void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) int nId1 = vRandom[nRndPos1]; int nId2 = vRandom[nRndPos2]; - assert(mapInfo.count(nId1) == 1); - assert(mapInfo.count(nId2) == 1); + const auto it_1{mapInfo.find(nId1)}; + const auto it_2{mapInfo.find(nId2)}; + assert(it_1 != mapInfo.end()); + assert(it_2 != mapInfo.end()); - mapInfo[nId1].nRandomPos = nRndPos2; - mapInfo[nId2].nRandomPos = nRndPos1; + it_1->second.nRandomPos = nRndPos2; + it_2->second.nRandomPos = nRndPos1; vRandom[nRndPos1] = nId2; vRandom[nRndPos2] = nId1; } -void CAddrMan::Delete(int nId) +void AddrManImpl::Delete(int nId) { AssertLockHeld(cs); assert(mapInfo.count(nId) != 0); - CAddrInfo& info = mapInfo[nId]; + AddrInfo& info = mapInfo[nId]; assert(!info.fInTried); assert(info.nRefCount == 0); @@ -176,33 +470,37 @@ void CAddrMan::Delete(int nId) nNew--; } -void CAddrMan::ClearNew(int nUBucket, int nUBucketPos) +void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) { AssertLockHeld(cs); // if there is an entry in the specified bucket, delete it. if (vvNew[nUBucket][nUBucketPos] != -1) { int nIdDelete = vvNew[nUBucket][nUBucketPos]; - CAddrInfo& infoDelete = mapInfo[nIdDelete]; + AddrInfo& infoDelete = mapInfo[nIdDelete]; assert(infoDelete.nRefCount > 0); infoDelete.nRefCount--; vvNew[nUBucket][nUBucketPos] = -1; + LogPrint(BCLog::ADDRMAN, "Removed %s from new[%i][%i]\n", infoDelete.ToString(), nUBucket, nUBucketPos); if (infoDelete.nRefCount == 0) { Delete(nIdDelete); } } } -void CAddrMan::MakeTried(CAddrInfo& info, int nId) +void AddrManImpl::MakeTried(AddrInfo& info, int nId) { AssertLockHeld(cs); // remove the entry from all new buckets - for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - int pos = info.GetBucketPosition(nKey, true, bucket); + const int start_bucket{info.GetNewBucket(nKey, m_netgroupman)}; + for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; ++n) { + const int bucket{(start_bucket + n) % ADDRMAN_NEW_BUCKET_COUNT}; + const int pos{info.GetBucketPosition(nKey, true, bucket)}; if (vvNew[bucket][pos] == nId) { vvNew[bucket][pos] = -1; info.nRefCount--; + if (info.nRefCount == 0) break; } } nNew--; @@ -210,7 +508,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey, m_asmap); + int nKBucket = info.GetTriedBucket(nKey, m_netgroupman); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -218,7 +516,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) // find an item to evict int nIdEvict = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nIdEvict) == 1); - CAddrInfo& infoOld = mapInfo[nIdEvict]; + AddrInfo& infoOld = mapInfo[nIdEvict]; // Remove the to-be-evicted item from the tried set. infoOld.fInTried = false; @@ -226,7 +524,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) nTried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); + int nUBucket = infoOld.GetNewBucket(nKey, m_netgroupman); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -235,6 +533,8 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) infoOld.nRefCount = 1; vvNew[nUBucket][nUBucketPos] = nIdEvict; nNew++; + LogPrint(BCLog::ADDRMAN, "Moved %s from tried[%i][%i] to new[%i][%i] to make space\n", + infoOld.ToString(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); } assert(vvTried[nKBucket][nKBucketPos] == -1); @@ -243,103 +543,36 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) info.fInTried = true; } -void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime) -{ - AssertLockHeld(cs); - - int nId; - - nLastGood = nTime; - - CAddrInfo* pinfo = Find(addr, &nId); - - // if not found, bail out - if (!pinfo) - return; - - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; - - // update info - info.nLastSuccess = nTime; - info.nLastTry = nTime; - info.nAttempts = 0; - // nTime is not updated here, to avoid leaking information about - // currently-connected peers. - - // if it is already in the tried set, don't do anything else - if (info.fInTried) - return; - - // find a bucket it is in now - int nRnd = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); - int nUBucket = -1; - for (unsigned int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { - int nB = (n + nRnd) % ADDRMAN_NEW_BUCKET_COUNT; - int nBpos = info.GetBucketPosition(nKey, true, nB); - if (vvNew[nB][nBpos] == nId) { - nUBucket = nB; - break; - } - } - - // if no bucket is found, something bad happened; - // TODO: maybe re-add the node, but for now, just bail out - if (nUBucket == -1) - return; - - // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey, m_asmap); - int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); - - // Will moving this address into tried evict another entry? - if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { - // Output the entry we'd be colliding with, for debugging purposes - auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); - LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table (%s), moving %s to m_tried_collisions=%d\n", colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", addr.ToString(), m_tried_collisions.size()); - if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { - m_tried_collisions.insert(nId); - } - } else { - LogPrint(BCLog::ADDRMAN, "Moving %s to tried\n", addr.ToString()); - - // move nId to the tried tables - MakeTried(info, nId); - } -} - -bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::chrono::seconds time_penalty) { AssertLockHeld(cs); if (!addr.IsRoutable()) return false; - bool fNew = false; int nId; - CAddrInfo* pinfo = Find(addr, &nId); + AddrInfo* pinfo = Find(addr, &nId); // Do not set a penalty for a source's self-announcement if (addr == source) { - nTimePenalty = 0; + time_penalty = 0s; } if (pinfo) { // periodically update nTime - bool fCurrentlyOnline = (GetAdjustedTime() - addr.nTime < 24 * 60 * 60); - int64_t nUpdateInterval = (fCurrentlyOnline ? 60 * 60 : 24 * 60 * 60); - if (addr.nTime && (!pinfo->nTime || pinfo->nTime < addr.nTime - nUpdateInterval - nTimePenalty)) - pinfo->nTime = std::max((int64_t)0, addr.nTime - nTimePenalty); + const bool currently_online{NodeClock::now() - addr.nTime < 24h}; + const auto update_interval{currently_online ? 1h : 24h}; + if (pinfo->nTime < addr.nTime - update_interval - time_penalty) { + pinfo->nTime = std::max(NodeSeconds{0s}, addr.nTime - time_penalty); + } // add services pinfo->nServices = ServiceFlags(pinfo->nServices | addr.nServices); // do not update if no new information is present - if (!addr.nTime || (pinfo->nTime && addr.nTime <= pinfo->nTime)) + if (addr.nTime <= pinfo->nTime) { return false; + } // do not update if the entry was already in the "tried" table if (pinfo->fInTried) @@ -357,17 +590,16 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP return false; } else { pinfo = Create(addr, source, &nId); - pinfo->nTime = std::max((int64_t)0, (int64_t)pinfo->nTime - nTimePenalty); + pinfo->nTime = std::max(NodeSeconds{0s}, pinfo->nTime - time_penalty); nNew++; - fNew = true; } - int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_netgroupman); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); + bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (vvNew[nUBucket][nUBucketPos] != nId) { - bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (!fInsert) { - CAddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; + AddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; if (infoExisting.IsTerrible() || (infoExisting.nRefCount > 1 && pinfo->nRefCount == 0)) { // Overwrite the existing new table entry. fInsert = true; @@ -377,48 +609,110 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP ClearNew(nUBucket, nUBucketPos); pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; + LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", + addr.ToString(), m_netgroupman.GetMappedAS(addr), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); } } } - return fNew; + return fInsert; } -void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) +bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, NodeSeconds time) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + int nId; + + m_last_good = time; + + AddrInfo* pinfo = Find(addr, &nId); // if not found, bail out - if (!pinfo) - return; + if (!pinfo) return false; + + AddrInfo& info = *pinfo; + + // update info + info.m_last_success = time; + info.m_last_try = time; + info.nAttempts = 0; + // nTime is not updated here, to avoid leaking information about + // currently-connected peers. + + // if it is already in the tried set, don't do anything else + if (info.fInTried) return false; + + // if it is not in new, something bad happened + if (!Assume(info.nRefCount > 0)) return false; + + + // which tried bucket to move the entry to + int tried_bucket = info.GetTriedBucket(nKey, m_netgroupman); + int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); + + // Will moving this address into tried evict another entry? + if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { + if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { + m_tried_collisions.insert(nId); + } + // Output the entry we'd be colliding with, for debugging purposes + auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); + LogPrint(BCLog::ADDRMAN, "Collision with %s while attempting to move %s to tried table. Collisions=%d\n", + colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", + addr.ToString(), + m_tried_collisions.size()); + return false; + } else { + // move nId to the tried tables + MakeTried(info, nId); + LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", + addr.ToString(), m_netgroupman.GetMappedAS(addr), tried_bucket, tried_bucket_pos); + return true; + } +} + +bool AddrManImpl::Add_(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) +{ + int added{0}; + for (std::vector::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) { + added += AddSingle(*it, source, time_penalty) ? 1 : 0; + } + if (added > 0) { + LogPrint(BCLog::ADDRMAN, "Added %i addresses (of %i) from %s: %i tried, %i new\n", added, vAddr.size(), source.ToString(), nTried, nNew); + } + return added > 0; +} + +void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, NodeSeconds time) +{ + AssertLockHeld(cs); - CAddrInfo& info = *pinfo; + AddrInfo* pinfo = Find(addr); - // check whether we are talking about the exact same CService (including same port) - if (info != addr) + // if not found, bail out + if (!pinfo) return; + AddrInfo& info = *pinfo; + // update info - info.nLastTry = nTime; - if (fCountFailure && info.nLastCountAttempt < nLastGood) { - info.nLastCountAttempt = nTime; + info.m_last_try = time; + if (fCountFailure && info.m_last_count_attempt < m_last_good) { + info.m_last_count_attempt = time; info.nAttempts++; } } -CAddrInfo CAddrMan::Select_(bool newOnly) +std::pair AddrManImpl::Select_(bool newOnly) const { AssertLockHeld(cs); - if (vRandom.empty()) - return CAddrInfo(); + if (vRandom.empty()) return {}; - if (newOnly && nNew == 0) - return CAddrInfo(); + if (newOnly && nNew == 0) return {}; // Use a 50% chance for choosing between tried and new table entries. if (!newOnly && @@ -426,120 +720,62 @@ CAddrInfo CAddrMan::Select_(bool newOnly) // use a tried node double fChanceFactor = 1.0; while (1) { + // Pick a tried bucket, and an initial position in that bucket. int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT); int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvTried[nKBucket][nKBucketPos] == -1) { - nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; - nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; + } + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE]; + const auto it_found{mapInfo.find(nId)}; + assert(it_found != mapInfo.end()); + const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToString()); + return {info, info.m_last_try}; } - int nId = vvTried[nKBucket][nKBucketPos]; - assert(mapInfo.count(nId) == 1); - CAddrInfo& info = mapInfo[nId]; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; + // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; } } else { // use a new node double fChanceFactor = 1.0; while (1) { + // Pick a new bucket, and an initial position in that bucket. int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvNew[nUBucket][nUBucketPos] == -1) { - nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; - nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; } - int nId = vvNew[nUBucket][nUBucketPos]; - assert(mapInfo.count(nId) == 1); - CAddrInfo& info = mapInfo[nId]; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; - fChanceFactor *= 1.2; - } - } -} - -#ifdef DEBUG_ADDRMAN -int CAddrMan::Check_() -{ - AssertLockHeld(cs); - - std::unordered_set setTried; - std::unordered_map mapNew; - - if (vRandom.size() != (size_t)(nTried + nNew)) - return -7; - - for (const auto& entry : mapInfo) { - int n = entry.first; - const CAddrInfo& info = entry.second; - if (info.fInTried) { - if (!info.nLastSuccess) - return -1; - if (info.nRefCount) - return -2; - setTried.insert(n); - } else { - if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) - return -3; - if (!info.nRefCount) - return -4; - mapNew[n] = info.nRefCount; - } - if (mapAddr[info] != n) - return -5; - if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) - return -14; - if (info.nLastTry < 0) - return -6; - if (info.nLastSuccess < 0) - return -8; - } - - if (setTried.size() != (size_t)nTried) - return -9; - if (mapNew.size() != (size_t)nNew) - return -10; - - for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvTried[n][i] != -1) { - if (!setTried.count(vvTried[n][i])) - return -11; - if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n) - return -17; - if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i) - return -18; - setTried.erase(vvTried[n][i]); - } - } - } - - for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[n][i] != -1) { - if (!mapNew.count(vvNew[n][i])) - return -12; - if (mapInfo[vvNew[n][i]].GetBucketPosition(nKey, true, n) != i) - return -19; - if (--mapNew[vvNew[n][i]] == 0) - mapNew.erase(vvNew[n][i]); + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE]; + const auto it_found{mapInfo.find(nId)}; + assert(it_found != mapInfo.end()); + const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToString()); + return {info, info.m_last_try}; } + // Otherwise start over with a (likely) different bucket, and increased chance factor. + fChanceFactor *= 1.2; } } - - if (setTried.size()) - return -13; - if (mapNew.size()) - return -15; - if (nKey.IsNull()) - return -16; - - return 0; } -#endif -void CAddrMan::GetAddr_(std::vector& vAddr, size_t max_addresses, size_t max_pct, std::optional network) +std::vector AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct, std::optional network) const { AssertLockHeld(cs); @@ -552,16 +788,18 @@ void CAddrMan::GetAddr_(std::vector& vAddr, size_t max_addresses, size } // gather a list of random nodes, skipping those of low quality - const int64_t now{GetAdjustedTime()}; + const auto now{Now()}; + std::vector addresses; for (unsigned int n = 0; n < vRandom.size(); n++) { - if (vAddr.size() >= nNodes) + if (addresses.size() >= nNodes) break; int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n; SwapRandom(n, nRndPos); - assert(mapInfo.count(vRandom[n]) == 1); + const auto it{mapInfo.find(vRandom[n])}; + assert(it != mapInfo.end()); - const CAddrInfo& ai = mapInfo[vRandom[n]]; + const AddrInfo& ai{it->second}; // Filter by network (optional) if (network != std::nullopt && ai.GetNetClass() != network) continue; @@ -569,53 +807,48 @@ void CAddrMan::GetAddr_(std::vector& vAddr, size_t max_addresses, size // Filter for quality if (ai.IsTerrible(now)) continue; - vAddr.push_back(ai); + addresses.push_back(ai); } + LogPrint(BCLog::ADDRMAN, "GetAddr returned %d random addresses\n", addresses.size()); + return addresses; } -void CAddrMan::Connected_(const CService& addr, int64_t nTime) +void AddrManImpl::Connected_(const CService& addr, NodeSeconds time) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; + AddrInfo& info = *pinfo; // update info - int64_t nUpdateInterval = 20 * 60; - if (nTime - info.nTime > nUpdateInterval) - info.nTime = nTime; + const auto update_interval{20min}; + if (time - info.nTime > update_interval) { + info.nTime = time; + } } -void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) +void AddrManImpl::SetServices_(const CService& addr, ServiceFlags nServices) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; + AddrInfo& info = *pinfo; // update info info.nServices = nServices; } -void CAddrMan::ResolveCollisions_() +void AddrManImpl::ResolveCollisions_() { AssertLockHeld(cs); @@ -628,10 +861,10 @@ void CAddrMan::ResolveCollisions_() if (mapInfo.count(id_new) != 1) { erase_collision = true; } else { - CAddrInfo& info_new = mapInfo[id_new]; + AddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. - int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); + int tried_bucket = info_new.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); if (!info_new.IsValid()) { // id_new may no longer map to a valid address erase_collision = true; @@ -639,31 +872,33 @@ void CAddrMan::ResolveCollisions_() // Get the to-be-evicted address that is being tested int id_old = vvTried[tried_bucket][tried_bucket_pos]; - CAddrInfo& info_old = mapInfo[id_old]; + AddrInfo& info_old = mapInfo[id_old]; + + const auto current_time{Now()}; // Has successfully connected in last X hours - if (GetAdjustedTime() - info_old.nLastSuccess < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { + if (current_time - info_old.m_last_success < ADDRMAN_REPLACEMENT) { erase_collision = true; - } else if (GetAdjustedTime() - info_old.nLastTry < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { // attempted to connect and failed in last X hours + } else if (current_time - info_old.m_last_try < ADDRMAN_REPLACEMENT) { // attempted to connect and failed in last X hours // Give address at least 60 seconds to successfully connect - if (GetAdjustedTime() - info_old.nLastTry > 60) { + if (current_time - info_old.m_last_try > 60s) { LogPrint(BCLog::ADDRMAN, "Replacing %s with %s in tried table\n", info_old.ToString(), info_new.ToString()); // Replaces an existing address already in the tried table with the new address - Good_(info_new, false, GetAdjustedTime()); + Good_(info_new, false, current_time); erase_collision = true; } - } else if (GetAdjustedTime() - info_new.nLastSuccess > ADDRMAN_TEST_WINDOW) { + } else if (current_time - info_new.m_last_success > ADDRMAN_TEST_WINDOW) { // If the collision hasn't resolved in some reasonable amount of time, // just evict the old entry -- we must not be able to // connect to it for some reason. LogPrint(BCLog::ADDRMAN, "Unable to test; replacing %s with %s in tried table anyway\n", info_old.ToString(), info_new.ToString()); - Good_(info_new, false, GetAdjustedTime()); + Good_(info_new, false, current_time); erase_collision = true; } } else { // Collision is not actually a collision anymore - Good_(info_new, false, GetAdjustedTime()); + Good_(info_new, false, Now()); erase_collision = true; } } @@ -676,11 +911,11 @@ void CAddrMan::ResolveCollisions_() } } -CAddrInfo CAddrMan::SelectTriedCollision_() +std::pair AddrManImpl::SelectTriedCollision_() { AssertLockHeld(cs); - if (m_tried_collisions.size() == 0) return CAddrInfo(); + if (m_tried_collisions.size() == 0) return {}; std::set::iterator it = m_tried_collisions.begin(); @@ -691,43 +926,317 @@ CAddrInfo CAddrMan::SelectTriedCollision_() // If id_new not found in mapInfo remove it from m_tried_collisions if (mapInfo.count(id_new) != 1) { m_tried_collisions.erase(it); - return CAddrInfo(); + return {}; } - const CAddrInfo& newInfo = mapInfo[id_new]; + const AddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to - int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); + int tried_bucket = newInfo.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); - int id_old = vvTried[tried_bucket][tried_bucket_pos]; + const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]]; + return {info_old, info_old.m_last_try}; +} + +std::optional AddrManImpl::FindAddressEntry_(const CAddress& addr) +{ + AssertLockHeld(cs); + + AddrInfo* addr_info = Find(addr); + + if (!addr_info) return std::nullopt; + + if(addr_info->fInTried) { + int bucket{addr_info->GetTriedBucket(nKey, m_netgroupman)}; + return AddressPosition(/*tried_in=*/true, + /*multiplicity_in=*/1, + /*bucket_in=*/bucket, + /*position_in=*/addr_info->GetBucketPosition(nKey, false, bucket)); + } else { + int bucket{addr_info->GetNewBucket(nKey, m_netgroupman)}; + return AddressPosition(/*tried_in=*/false, + /*multiplicity_in=*/addr_info->nRefCount, + /*bucket_in=*/bucket, + /*position_in=*/addr_info->GetBucketPosition(nKey, true, bucket)); + } +} + +void AddrManImpl::Check() const +{ + AssertLockHeld(cs); + + // Run consistency checks 1 in m_consistency_check_ratio times if enabled + if (m_consistency_check_ratio == 0) return; + if (insecure_rand.randrange(m_consistency_check_ratio) >= 1) return; - return mapInfo[id_old]; + const int err{CheckAddrman()}; + if (err) { + LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); + assert(false); + } } -std::vector CAddrMan::DecodeAsmap(fs::path path) +int AddrManImpl::CheckAddrman() const { - std::vector bits; - FILE *filestr = fsbridge::fopen(path, "rb"); - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - if (file.IsNull()) { - LogPrintf("Failed to open asmap file from disk\n"); - return bits; + AssertLockHeld(cs); + + LOG_TIME_MILLIS_WITH_CATEGORY_MSG_ONCE( + strprintf("new %i, tried %i, total %u", nNew, nTried, vRandom.size()), BCLog::ADDRMAN); + + std::unordered_set setTried; + std::unordered_map mapNew; + + if (vRandom.size() != (size_t)(nTried + nNew)) + return -7; + + for (const auto& entry : mapInfo) { + int n = entry.first; + const AddrInfo& info = entry.second; + if (info.fInTried) { + if (!TicksSinceEpoch(info.m_last_success)) { + return -1; + } + if (info.nRefCount) + return -2; + setTried.insert(n); + } else { + if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) + return -3; + if (!info.nRefCount) + return -4; + mapNew[n] = info.nRefCount; + } + const auto it{mapAddr.find(info)}; + if (it == mapAddr.end() || it->second != n) { + return -5; + } + if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) + return -14; + if (info.m_last_try < NodeSeconds{0s}) { + return -6; + } + if (info.m_last_success < NodeSeconds{0s}) { + return -8; + } } - fseek(filestr, 0, SEEK_END); - int length = ftell(filestr); - LogPrintf("Opened asmap file %s (%d bytes) from disk\n", path, length); - fseek(filestr, 0, SEEK_SET); - uint8_t cur_byte; - for (int i = 0; i < length; ++i) { - file >> cur_byte; - for (int bit = 0; bit < 8; ++bit) { - bits.push_back((cur_byte >> bit) & 1); + + if (setTried.size() != (size_t)nTried) + return -9; + if (mapNew.size() != (size_t)nNew) + return -10; + + for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvTried[n][i] != -1) { + if (!setTried.count(vvTried[n][i])) + return -11; + const auto it{mapInfo.find(vvTried[n][i])}; + if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_netgroupman) != n) { + return -17; + } + if (it->second.GetBucketPosition(nKey, false, n) != i) { + return -18; + } + setTried.erase(vvTried[n][i]); + } } } - if (!SanityCheckASMap(bits)) { - LogPrintf("Sanity check of asmap file %s failed\n", path); - return {}; + + for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[n][i] != -1) { + if (!mapNew.count(vvNew[n][i])) + return -12; + const auto it{mapInfo.find(vvNew[n][i])}; + if (it == mapInfo.end() || it->second.GetBucketPosition(nKey, true, n) != i) { + return -19; + } + if (--mapNew[vvNew[n][i]] == 0) + mapNew.erase(vvNew[n][i]); + } + } } - return bits; + + if (setTried.size()) + return -13; + if (mapNew.size()) + return -15; + if (nKey.IsNull()) + return -16; + + return 0; +} + +size_t AddrManImpl::size() const +{ + LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead + return vRandom.size(); +} + +bool AddrManImpl::Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) +{ + LOCK(cs); + Check(); + auto ret = Add_(vAddr, source, time_penalty); + Check(); + return ret; +} + +bool AddrManImpl::Good(const CService& addr, NodeSeconds time) +{ + LOCK(cs); + Check(); + auto ret = Good_(addr, /*test_before_evict=*/true, time); + Check(); + return ret; +} + +void AddrManImpl::Attempt(const CService& addr, bool fCountFailure, NodeSeconds time) +{ + LOCK(cs); + Check(); + Attempt_(addr, fCountFailure, time); + Check(); +} + +void AddrManImpl::ResolveCollisions() +{ + LOCK(cs); + Check(); + ResolveCollisions_(); + Check(); +} + +std::pair AddrManImpl::SelectTriedCollision() +{ + LOCK(cs); + Check(); + const auto ret = SelectTriedCollision_(); + Check(); + return ret; +} + +std::pair AddrManImpl::Select(bool newOnly) const +{ + LOCK(cs); + Check(); + const auto addrRet = Select_(newOnly); + Check(); + return addrRet; +} + +std::vector AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const +{ + LOCK(cs); + Check(); + const auto addresses = GetAddr_(max_addresses, max_pct, network); + Check(); + return addresses; +} + +void AddrManImpl::Connected(const CService& addr, NodeSeconds time) +{ + LOCK(cs); + Check(); + Connected_(addr, time); + Check(); +} + +void AddrManImpl::SetServices(const CService& addr, ServiceFlags nServices) +{ + LOCK(cs); + Check(); + SetServices_(addr, nServices); + Check(); +} + +std::optional AddrManImpl::FindAddressEntry(const CAddress& addr) +{ + LOCK(cs); + Check(); + auto entry = FindAddressEntry_(addr); + Check(); + return entry; +} + +AddrMan::AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio) + : m_impl(std::make_unique(netgroupman, deterministic, consistency_check_ratio)) {} + +AddrMan::~AddrMan() = default; + +template +void AddrMan::Serialize(Stream& s_) const +{ + m_impl->Serialize(s_); +} + +template +void AddrMan::Unserialize(Stream& s_) +{ + m_impl->Unserialize(s_); +} + +// explicit instantiation +template void AddrMan::Serialize(CHashWriter& s) const; +template void AddrMan::Serialize(CAutoFile& s) const; +template void AddrMan::Serialize(CDataStream& s) const; +template void AddrMan::Unserialize(CAutoFile& s); +template void AddrMan::Unserialize(CHashVerifier& s); +template void AddrMan::Unserialize(CDataStream& s); +template void AddrMan::Unserialize(CHashVerifier& s); + +size_t AddrMan::size() const +{ + return m_impl->size(); +} + +bool AddrMan::Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) +{ + return m_impl->Add(vAddr, source, time_penalty); +} + +bool AddrMan::Good(const CService& addr, NodeSeconds time) +{ + return m_impl->Good(addr, time); +} + +void AddrMan::Attempt(const CService& addr, bool fCountFailure, NodeSeconds time) +{ + m_impl->Attempt(addr, fCountFailure, time); +} + +void AddrMan::ResolveCollisions() +{ + m_impl->ResolveCollisions(); +} + +std::pair AddrMan::SelectTriedCollision() +{ + return m_impl->SelectTriedCollision(); +} + +std::pair AddrMan::Select(bool newOnly) const +{ + return m_impl->Select(newOnly); +} + +std::vector AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const +{ + return m_impl->GetAddr(max_addresses, max_pct, network); +} + +void AddrMan::Connected(const CService& addr, NodeSeconds time) +{ + m_impl->Connected(addr, time); +} + +void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) +{ + m_impl->SetServices(addr, nServices); +} + +std::optional AddrMan::FindAddressEntry(const CAddress& addr) +{ + return m_impl->FindAddressEntry(addr); } diff --git a/src/addrman.h b/src/addrman.h index 6f081b8dc14e..5099c8c7a37e 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -1,100 +1,57 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2020 The Bitcoin Core developers +// Copyright (c) 2012-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H -#include -#include -#include -#include #include +#include #include -#include #include -#include -#include -#include -#include +#include -#include +#include +#include #include -#include -#include -#include +#include #include -/** - * Extended statistics about a CAddress - */ -class CAddrInfo : public CAddress +class InvalidAddrManVersionError : public std::ios_base::failure { public: - //! last try whatsoever by us (memory only) - int64_t nLastTry{0}; - - //! last counted attempt (memory only) - int64_t nLastCountAttempt{0}; - -private: - //! where knowledge about this address first came from - CNetAddr source; - - //! last successful connection by us - int64_t nLastSuccess{0}; - - //! connection attempts since last successful attempt - int nAttempts{0}; - - //! reference count in new sets (memory only) - int nRefCount{0}; - - //! in tried set? (memory only) - bool fInTried{false}; - - //! position in vRandom - int nRandomPos{-1}; - - friend class CAddrMan; - -public: + InvalidAddrManVersionError(std::string msg) : std::ios_base::failure(msg) { } +}; - SERIALIZE_METHODS(CAddrInfo, obj) - { - READWRITEAS(CAddress, obj); - READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); - } +class AddrManImpl; - CAddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) - { - } +/** Default for -checkaddrman */ +static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0}; - CAddrInfo() : CAddress(), source() - { - } +/** Test-only struct, capturing info about an address in AddrMan */ +struct AddressPosition { + // Whether the address is in the new or tried table + const bool tried; - //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey, const std::vector &asmap) const; + // Addresses in the tried table should always have a multiplicity of 1. + // Addresses in the new table can have multiplicity between 1 and + // ADDRMAN_NEW_BUCKETS_PER_ADDRESS + const int multiplicity; - //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector &asmap) const; + // If the address is in the new table, the bucket and position are + // populated based on the first source who sent the address. + // In certain edge cases, this may not be where the address is currently + // located. + const int bucket; + const int position; - //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey, const std::vector &asmap) const - { - return GetNewBucket(nKey, source, asmap); + bool operator==(AddressPosition other) { + return std::tie(tried, multiplicity, bucket, position) == + std::tie(other.tried, other.multiplicity, other.bucket, other.position); } - - //! Calculate in which position of a bucket to store this entry. - int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; - - //! Determine whether the statistics about this entry are bad enough so that it can just be deleted - bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; - - //! Calculate the relative chance this entry should be given when selecting nodes to connect to - double GetChance(int64_t nNow = GetAdjustedTime()) const; + explicit AddressPosition(bool tried_in, int multiplicity_in, int bucket_in, int position_in) + : tried{tried_in}, multiplicity{multiplicity_in}, bucket{bucket_in}, position{position_in} {} }; /** Stochastic address manager @@ -104,655 +61,101 @@ class CAddrInfo : public CAddress * * Make sure no (localized) attacker can fill the entire table with his nodes/addresses. * * To that end: - * * Addresses are organized into buckets. - * * Addresses that have not yet been tried go into 1024 "new" buckets. - * * Based on the address range (/16 for IPv4) of the source of information, 64 buckets are selected at random. + * * Addresses are organized into buckets that can each store up to 64 entries. + * * Addresses to which our node has not successfully connected go into 1024 "new" buckets. + * * Based on the address range (/16 for IPv4) of the source of information, or if an asmap is provided, + * the AS it belongs to (for IPv4/IPv6), 64 buckets are selected at random. * * The actual bucket is chosen from one of these, based on the range in which the address itself is located. + * * The position in the bucket is chosen based on the full address. * * One single address can occur in up to 8 different buckets to increase selection chances for addresses that * are seen frequently. The chance for increasing this multiplicity decreases exponentially. - * * When adding a new address to a full bucket, a randomly chosen entry (with a bias favoring less recently seen - * ones) is removed from it first. + * * When adding a new address to an occupied position of a bucket, it will not replace the existing entry + * unless that address is also stored in another bucket or it doesn't meet one of several quality criteria + * (see IsTerrible for exact criteria). * * Addresses of nodes that are known to be accessible go into 256 "tried" buckets. * * Each address range selects at random 8 of these buckets. * * The actual bucket is chosen from one of these, based on the full address. - * * When adding a new good address to a full bucket, a randomly chosen entry (with a bias favoring less recently - * tried ones) is evicted from it, back to the "new" buckets. + * * When adding a new good address to an occupied position of a bucket, a FEELER connection to the + * old address is attempted. The old entry is only replaced and moved back to the "new" buckets if this + * attempt was unsuccessful. * * Bucket selection is based on cryptographic hashing, using a randomly-generated 256-bit key, which should not * be observable by adversaries. - * * Several indexes are kept for high performance. Defining DEBUG_ADDRMAN will introduce frequent (and expensive) - * consistency checks for the entire data structure. + * * Several indexes are kept for high performance. Setting m_consistency_check_ratio with the -checkaddrman + * configuration option will introduce (expensive) consistency checks for the entire data structure. */ - -//! total number of buckets for tried addresses -#define ADDRMAN_TRIED_BUCKET_COUNT_LOG2 8 - -//! total number of buckets for new addresses -#define ADDRMAN_NEW_BUCKET_COUNT_LOG2 10 - -//! maximum allowed number of entries in buckets for new and tried addresses -#define ADDRMAN_BUCKET_SIZE_LOG2 6 - -//! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread -#define ADDRMAN_TRIED_BUCKETS_PER_GROUP 8 - -//! over how many buckets entries with new addresses originating from a single group are spread -#define ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP 64 - -//! in how many buckets for entries with new addresses a single address may occur -#define ADDRMAN_NEW_BUCKETS_PER_ADDRESS 8 - -//! how old addresses can maximally be -#define ADDRMAN_HORIZON_DAYS 30 - -//! after how many failed attempts we give up on a new node -#define ADDRMAN_RETRIES 3 - -//! how many successive failures are allowed ... -#define ADDRMAN_MAX_FAILURES 10 - -//! ... in at least this many days -#define ADDRMAN_MIN_FAIL_DAYS 7 - -//! how recent a successful connection should be before we allow an address to be evicted from tried -#define ADDRMAN_REPLACEMENT_HOURS 4 - -//! Convenience -#define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2) -#define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2) -#define ADDRMAN_BUCKET_SIZE (1 << ADDRMAN_BUCKET_SIZE_LOG2) - -//! the maximum number of tried addr collisions to store -#define ADDRMAN_SET_TRIED_COLLISION_SIZE 10 - -//! the maximum time we'll spend trying to resolve a tried table collision, in seconds -static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes - -/** - * Stochastical (IP) address manager - */ -class CAddrMan +class AddrMan { +protected: + const std::unique_ptr m_impl; + public: - // Compressed IP->ASN mapping, loaded from a file when a node starts. - // Should be always empty if no file was provided. - // This mapping is then used for bucketing nodes in Addrman. - // - // If asmap is provided, nodes will be bucketed by - // AS they belong to, in order to make impossible for a node - // to connect to several nodes hosted in a single AS. - // This is done in response to Erebus attack, but also to generally - // diversify the connections every node creates, - // especially useful when a large fraction of nodes - // operate under a couple of cloud providers. - // - // If a new asmap was provided, the existing records - // would be re-bucketed accordingly. - std::vector m_asmap; + explicit AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio); - // Read asmap from provided binary file - static std::vector DecodeAsmap(fs::path path); + ~AddrMan(); - /** - * Serialized format. - * * format version byte (@see `Format`) - * * lowest compatible format version byte. This is used to help old software decide - * whether to parse the file. For example: - * * Bitcoin Core version N knows how to parse up to format=3. If a new format=4 is - * introduced in version N+1 that is compatible with format=3 and it is known that - * version N will be able to parse it, then version N+1 will write - * (format=4, lowest_compatible=3) in the first two bytes of the file, and so - * version N will still try to parse it. - * * Bitcoin Core version N+2 introduces a new incompatible format=5. It will write - * (format=5, lowest_compatible=5) and so any versions that do not know how to parse - * format=5 will not try to read the file. - * * nKey - * * nNew - * * nTried - * * number of "new" buckets XOR 2**30 - * * all new addresses (total count: nNew) - * * all tried addresses (total count: nTried) - * * for each new bucket: - * * number of elements - * * for each element: index in the serialized "all new addresses" - * * asmap checksum - * - * 2**30 is xorred with the number of buckets to make addrman deserializer v0 detect it - * as incompatible. This is necessary because it did not check the version number on - * deserialization. - * - * vvNew, vvTried, mapInfo, mapAddr and vRandom are never encoded explicitly; - * they are instead reconstructed from the other information. - * - * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports - * changes to the ADDRMAN_ parameters without breaking the on-disk structure. - * - * We don't use SERIALIZE_METHODS since the serialization and deserialization code has - * very little in common. - */ template - void Serialize(Stream& s_) const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - - // Always serialize in the latest version (FILE_FORMAT). - - OverrideStream s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT); - - s << static_cast(FILE_FORMAT); - - // Increment `lowest_compatible` iff a newly introduced format is incompatible with - // the previous one. - static constexpr uint8_t lowest_compatible = Format::V3_BIP155; - s << static_cast(INCOMPATIBILITY_BASE + lowest_compatible); - - s << nKey; - s << nNew; - s << nTried; - - int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); - s << nUBuckets; - std::unordered_map mapUnkIds; - int nIds = 0; - for (const auto& entry : mapInfo) { - mapUnkIds[entry.first] = nIds; - const CAddrInfo &info = entry.second; - if (info.nRefCount) { - assert(nIds != nNew); // this means nNew was wrong, oh ow - s << info; - nIds++; - } - } - nIds = 0; - for (const auto& entry : mapInfo) { - const CAddrInfo &info = entry.second; - if (info.fInTried) { - assert(nIds != nTried); // this means nTried was wrong, oh ow - s << info; - nIds++; - } - } - for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - int nSize = 0; - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[bucket][i] != -1) - nSize++; - } - s << nSize; - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[bucket][i] != -1) { - int nIndex = mapUnkIds[vvNew[bucket][i]]; - s << nIndex; - } - } - } - // Store asmap checksum after bucket entries so that it - // can be ignored by older clients for backward compatibility. - uint256 asmap_checksum; - if (m_asmap.size() != 0) { - asmap_checksum = SerializeHash(m_asmap); - } - s << asmap_checksum; - } + void Serialize(Stream& s_) const; template - void Unserialize(Stream& s_) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - - assert(vRandom.empty()); - - Format format; - s_ >> Using>(format); - - int stream_version = s_.GetVersion(); - if (format >= Format::V3_BIP155) { - // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress - // unserialize methods know that an address in addrv2 format is coming. - stream_version |= ADDRV2_FORMAT; - } - - OverrideStream s(&s_, s_.GetType(), stream_version); - - uint8_t compat; - s >> compat; - const uint8_t lowest_compatible = compat - INCOMPATIBILITY_BASE; - if (lowest_compatible > FILE_FORMAT) { - throw std::ios_base::failure(strprintf( - "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " - "but the maximum supported by this version of %s is %u.", - format, lowest_compatible, PACKAGE_NAME, static_cast(FILE_FORMAT))); - } - - s >> nKey; - s >> nNew; - s >> nTried; - int nUBuckets = 0; - s >> nUBuckets; - if (format >= Format::V1_DETERMINISTIC) { - nUBuckets ^= (1 << 30); - } - - if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nNew < 0) { - throw std::ios_base::failure( - strprintf("Corrupt CAddrMan serialization: nNew=%d, should be in [0, %u]", - nNew, - ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); - } - - if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nTried < 0) { - throw std::ios_base::failure( - strprintf("Corrupt CAddrMan serialization: nTried=%d, should be in [0, %u]", - nTried, - ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); - } - - // Deserialize entries from the new table. - for (int n = 0; n < nNew; n++) { - CAddrInfo &info = mapInfo[n]; - s >> info; - mapAddr[info] = n; - info.nRandomPos = vRandom.size(); - vRandom.push_back(n); - } - nIdCount = nNew; - - // Deserialize entries from the tried table. - int nLost = 0; - for (int n = 0; n < nTried; n++) { - CAddrInfo info; - s >> info; - int nKBucket = info.GetTriedBucket(nKey, m_asmap); - int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); - if (vvTried[nKBucket][nKBucketPos] == -1) { - info.nRandomPos = vRandom.size(); - info.fInTried = true; - vRandom.push_back(nIdCount); - mapInfo[nIdCount] = info; - mapAddr[info] = nIdCount; - vvTried[nKBucket][nKBucketPos] = nIdCount; - nIdCount++; - } else { - nLost++; - } - } - nTried -= nLost; - - // Store positions in the new table buckets to apply later (if possible). - // An entry may appear in up to ADDRMAN_NEW_BUCKETS_PER_ADDRESS buckets, - // so we store all bucket-entry_index pairs to iterate through later. - std::vector> bucket_entries; - - for (int bucket = 0; bucket < nUBuckets; ++bucket) { - int num_entries{0}; - s >> num_entries; - for (int n = 0; n < num_entries; ++n) { - int entry_index{0}; - s >> entry_index; - if (entry_index >= 0 && entry_index < nNew) { - bucket_entries.emplace_back(bucket, entry_index); - } - } - } - - // If the bucket count and asmap checksum haven't changed, then attempt - // to restore the entries to the buckets/positions they were in before - // serialization. - uint256 supplied_asmap_checksum; - if (m_asmap.size() != 0) { - supplied_asmap_checksum = SerializeHash(m_asmap); - } - uint256 serialized_asmap_checksum; - if (format >= Format::V2_ASMAP) { - s >> serialized_asmap_checksum; - } - const bool restore_bucketing{nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && - serialized_asmap_checksum == supplied_asmap_checksum}; - - if (!restore_bucketing) { - LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); - } - - for (auto bucket_entry : bucket_entries) { - int bucket{bucket_entry.first}; - const int entry_index{bucket_entry.second}; - CAddrInfo& info = mapInfo[entry_index]; - - // The entry shouldn't appear in more than - // ADDRMAN_NEW_BUCKETS_PER_ADDRESS. If it has already, just skip - // this bucket_entry. - if (info.nRefCount >= ADDRMAN_NEW_BUCKETS_PER_ADDRESS) continue; - - int bucket_position = info.GetBucketPosition(nKey, true, bucket); - if (restore_bucketing && vvNew[bucket][bucket_position] == -1) { - // Bucketing has not changed, using existing bucket positions for the new table - vvNew[bucket][bucket_position] = entry_index; - ++info.nRefCount; - } else { - // In case the new table data cannot be used (bucket count wrong or new asmap), - // try to give them a reference based on their primary source address. - bucket = info.GetNewBucket(nKey, m_asmap); - bucket_position = info.GetBucketPosition(nKey, true, bucket); - if (vvNew[bucket][bucket_position] == -1) { - vvNew[bucket][bucket_position] = entry_index; - ++info.nRefCount; - } - } - } - - // Prune new entries with refcount 0 (as a result of collisions). - int nLostUnk = 0; - for (auto it = mapInfo.cbegin(); it != mapInfo.cend(); ) { - if (it->second.fInTried == false && it->second.nRefCount == 0) { - const auto itCopy = it++; - Delete(itCopy->first); - ++nLostUnk; - } else { - ++it; - } - } - if (nLost + nLostUnk > 0) { - LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions\n", nLostUnk, nLost); - } - - RemoveInvalid(); - - Check(); - } - - void Clear() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - std::vector().swap(vRandom); - nKey = insecure_rand.rand256(); - for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { - vvNew[bucket][entry] = -1; - } - } - for (size_t bucket = 0; bucket < ADDRMAN_TRIED_BUCKET_COUNT; bucket++) { - for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { - vvTried[bucket][entry] = -1; - } - } - - nIdCount = 0; - nTried = 0; - nNew = 0; - nLastGood = 1; //Initially at 1 so that "never" is strictly worse. - mapInfo.clear(); - mapAddr.clear(); - } - - CAddrMan() - { - Clear(); - } - - ~CAddrMan() - { - nKey.SetNull(); - } + void Unserialize(Stream& s_); //! Return the number of (unique) addresses in all tables. - size_t size() const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead - return vRandom.size(); - } - - //! Add a single address. - bool Add(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty = 0) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - bool fRet = false; - Check(); - fRet |= Add_(addr, source, nTimePenalty); - Check(); - if (fRet) { - LogPrint(BCLog::ADDRMAN, "Added %s from %s: %i tried, %i new\n", addr.ToStringIPPort(), source.ToString(), nTried, nNew); - } - return fRet; - } + size_t size() const; - //! Add multiple addresses. - bool Add(const std::vector &vAddr, const CNetAddr& source, int64_t nTimePenalty = 0) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - int nAdd = 0; - Check(); - for (std::vector::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) - nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; - Check(); - if (nAdd) { - LogPrint(BCLog::ADDRMAN, "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); - } - return nAdd > 0; - } + /** + * Attempt to add one or more addresses to addrman's new table. + * + * @param[in] vAddr Address records to attempt to add. + * @param[in] source The address of the node that sent us these addr records. + * @param[in] time_penalty A "time penalty" to apply to the address record's nTime. If a peer + * sends us an address record with nTime=n, then we'll add it to our + * addrman with nTime=(n - time_penalty). + * @return true if at least one address is successfully added. */ + bool Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty = 0s); - //! Mark an entry as accessible. - void Good(const CService &addr, bool test_before_evict = true, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Good_(addr, test_before_evict, nTime); - Check(); - } + /** + * Mark an address record as accessible and attempt to move it to addrman's tried table. + * + * @param[in] addr Address record to attempt to move to tried table. + * @param[in] time The time that we were last connected to this peer. + * @return true if the address is successfully moved from the new table to the tried table. + */ + bool Good(const CService& addr, NodeSeconds time = Now()); //! Mark an entry as connection attempted to. - void Attempt(const CService &addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Attempt_(addr, fCountFailure, nTime); - Check(); - } + void Attempt(const CService& addr, bool fCountFailure, NodeSeconds time = Now()); //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - ResolveCollisions_(); - Check(); - } - - //! Randomly select an address in tried that another address is attempting to evict. - CAddrInfo SelectTriedCollision() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo ret = SelectTriedCollision_(); - Check(); - return ret; - } + void ResolveCollisions(); /** - * Choose an address to connect to. + * Randomly select an address in the tried table that another address is + * attempting to evict. + * + * @return CAddress The record for the selected tried peer. + * seconds The last time we attempted to connect to that peer. */ - CAddrInfo Select(bool newOnly = false) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo addrRet = Select_(newOnly); - Check(); - return addrRet; - } + std::pair SelectTriedCollision(); /** - * Return all or many randomly selected addresses, optionally by network. + * Choose an address to connect to. * - * @param[in] max_addresses Maximum number of addresses to return (0 = all). - * @param[in] max_pct Maximum percentage of addresses to return (0 = all). - * @param[in] network Select only addresses of this network (nullopt = all). + * @param[in] newOnly Whether to only select addresses from the new table. + * @return CAddress The record for the selected peer. + * seconds The last time we attempted to connect to that peer. */ - std::vector GetAddr(size_t max_addresses, size_t max_pct, std::optional network) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - std::vector vAddr; - GetAddr_(vAddr, max_addresses, max_pct, network); - Check(); - return vAddr; - } - - //! Outer function for Connected_() - void Connected(const CService &addr, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Connected_(addr, nTime); - Check(); - } - - void SetServices(const CService &addr, ServiceFlags nServices) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - SetServices_(addr, nServices); - Check(); - } - -protected: - //! secret key to randomize bucket select with - uint256 nKey; - - //! Source of random numbers for randomization in inner loops - FastRandomContext insecure_rand; - -private: - //! A mutex to protect the inner data structures. - mutable Mutex cs; - - //! Serialization versions. - enum Format : uint8_t { - V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 - V1_DETERMINISTIC = 1, //!< for pre-asmap files - V2_ASMAP = 2, //!< for files including asmap version - V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format - }; - - //! The maximum format this software knows it can unserialize. Also, we always serialize - //! in this format. - //! The format (first byte in the serialized stream) can be higher than this and - //! still this software may be able to unserialize the file - if the second byte - //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. - static constexpr Format FILE_FORMAT = Format::V3_BIP155; - - //! The initial value of a field that is incremented every time an incompatible format - //! change is made (such that old software versions would not be able to parse and - //! understand the new file format). This is 32 because we overtook the "key size" - //! field which was 32 historically. - //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. - static constexpr uint8_t INCOMPATIBILITY_BASE = 32; - - //! last used nId - int nIdCount GUARDED_BY(cs); - - //! table with information about all nIds - std::unordered_map mapInfo GUARDED_BY(cs); - - //! find an nId based on its network address - std::unordered_map mapAddr GUARDED_BY(cs); - - //! randomly-ordered vector of all nIds - std::vector vRandom GUARDED_BY(cs); - - // number of "tried" entries - int nTried GUARDED_BY(cs); - - //! list of "tried" buckets - int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! number of (unique) "new" entries - int nNew GUARDED_BY(cs); - - //! list of "new" buckets - int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! last time Good was called (memory only) - int64_t nLastGood GUARDED_BY(cs); - - //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. - std::set m_tried_collisions; - - //! Find an entry. - CAddrInfo* Find(const CNetAddr& addr, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! find an entry, creating it if necessary. - //! nTime and nServices of the found node are updated, if necessary. - CAddrInfo* Create(const CAddress &addr, const CNetAddr &addrSource, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Swap two elements in vRandom. - void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Move an entry from the "new" table(s) to the "tried" table - void MakeTried(CAddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Delete an entry. It must not be in tried, and have refcount 0. - void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Clear a position in a "new" table. This is the only place where entries are actually deleted. - void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry "good", possibly moving it from "new" to "tried". - void Good_(const CService &addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Add an entry to the "new" table. - bool Add_(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry as attempted to connect. - void Attempt_(const CService &addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Select an address to connect to, if newOnly is set to true, only the new table is selected from. - CAddrInfo Select_(bool newOnly) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Return a random to-be-evicted tried table address. - CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Consistency check - void Check() - EXCLUSIVE_LOCKS_REQUIRED(cs) - { -#ifdef DEBUG_ADDRMAN - AssertLockHeld(cs); - const int err = Check_(); - if (err) { - LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); - } -#endif - } - -#ifdef DEBUG_ADDRMAN - //! Perform consistency check. Returns an error code or zero. - int Check_() EXCLUSIVE_LOCKS_REQUIRED(cs); -#endif + std::pair Select(bool newOnly = false) const; /** * Return all or many randomly selected addresses, optionally by network. * - * @param[out] vAddr Vector of randomly selected addresses from vRandom. * @param[in] max_addresses Maximum number of addresses to return (0 = all). * @param[in] max_pct Maximum percentage of addresses to return (0 = all). * @param[in] network Select only addresses of this network (nullopt = all). + * + * @return A vector of randomly selected addresses from vRandom. */ - void GetAddr_(std::vector& vAddr, size_t max_addresses, size_t max_pct, std::optional network) EXCLUSIVE_LOCKS_REQUIRED(cs); + std::vector GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const; /** We have successfully connected to this peer. Calling this function * updates the CAddress's nTime, which is used in our IsTerrible() @@ -763,17 +166,21 @@ class CAddrMan * not leak information about currently connected peers. * * @param[in] addr The address of the peer we were connected to - * @param[in] nTime The time that we were last connected to this peer + * @param[in] time The time that we were last connected to this peer */ - void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Connected(const CService& addr, NodeSeconds time = Now()); //! Update an entry's service bits. - void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Remove invalid addresses. - void RemoveInvalid() EXCLUSIVE_LOCKS_REQUIRED(cs); - - friend class CAddrManTest; + void SetServices(const CService& addr, ServiceFlags nServices); + + /** Test-only function + * Find the address record in AddrMan and return information about its + * position. + * @param[in] addr The address record to look up. + * @return Information about the address record in AddrMan + * or nullopt if address is not found. + */ + std::optional FindAddressEntry(const CAddress& addr); }; #endif // BITCOIN_ADDRMAN_H diff --git a/src/addrman_impl.h b/src/addrman_impl.h new file mode 100644 index 000000000000..376e79f49f5e --- /dev/null +++ b/src/addrman_impl.h @@ -0,0 +1,269 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRMAN_IMPL_H +#define BITCOIN_ADDRMAN_IMPL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/** Total number of buckets for tried addresses */ +static constexpr int32_t ADDRMAN_TRIED_BUCKET_COUNT_LOG2{8}; +static constexpr int ADDRMAN_TRIED_BUCKET_COUNT{1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2}; +/** Total number of buckets for new addresses */ +static constexpr int32_t ADDRMAN_NEW_BUCKET_COUNT_LOG2{10}; +static constexpr int ADDRMAN_NEW_BUCKET_COUNT{1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2}; +/** Maximum allowed number of entries in buckets for new and tried addresses */ +static constexpr int32_t ADDRMAN_BUCKET_SIZE_LOG2{6}; +static constexpr int ADDRMAN_BUCKET_SIZE{1 << ADDRMAN_BUCKET_SIZE_LOG2}; + +/** + * Extended statistics about a CAddress + */ +class AddrInfo : public CAddress +{ +public: + //! last try whatsoever by us (memory only) + NodeSeconds m_last_try{0s}; + + //! last counted attempt (memory only) + NodeSeconds m_last_count_attempt{0s}; + + //! where knowledge about this address first came from + CNetAddr source; + + //! last successful connection by us + NodeSeconds m_last_success{0s}; + + //! connection attempts since last successful attempt + int nAttempts{0}; + + //! reference count in new sets (memory only) + int nRefCount{0}; + + //! in tried set? (memory only) + bool fInTried{false}; + + //! position in vRandom + mutable int nRandomPos{-1}; + + SERIALIZE_METHODS(AddrInfo, obj) + { + READWRITEAS(CAddress, obj); + READWRITE(obj.source, Using>(obj.m_last_success), obj.nAttempts); + } + + AddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) + { + } + + AddrInfo() : CAddress(), source() + { + } + + //! Calculate in which "tried" bucket this entry belongs + int GetTriedBucket(const uint256& nKey, const NetGroupManager& netgroupman) const; + + //! Calculate in which "new" bucket this entry belongs, given a certain source + int GetNewBucket(const uint256& nKey, const CNetAddr& src, const NetGroupManager& netgroupman) const; + + //! Calculate in which "new" bucket this entry belongs, using its default source + int GetNewBucket(const uint256& nKey, const NetGroupManager& netgroupman) const + { + return GetNewBucket(nKey, source, netgroupman); + } + + //! Calculate in which position of a bucket to store this entry. + int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; + + //! Determine whether the statistics about this entry are bad enough so that it can just be deleted + bool IsTerrible(NodeSeconds now = Now()) const; + + //! Calculate the relative chance this entry should be given when selecting nodes to connect to + double GetChance(NodeSeconds now = Now()) const; +}; + +class AddrManImpl +{ +public: + AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio); + + ~AddrManImpl(); + + template + void Serialize(Stream& s_) const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + template + void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); + + size_t size() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + bool Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + bool Good(const CService& addr, NodeSeconds time) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Attempt(const CService& addr, bool fCountFailure, NodeSeconds time) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void ResolveCollisions() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair Select(bool newOnly) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::vector GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Connected(const CService& addr, NodeSeconds time) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void SetServices(const CService& addr, ServiceFlags nServices) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::optional FindAddressEntry(const CAddress& addr) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + friend class AddrManDeterministic; + +private: + //! A mutex to protect the inner data structures. + mutable Mutex cs; + + //! Source of random numbers for randomization in inner loops + mutable FastRandomContext insecure_rand GUARDED_BY(cs); + + //! secret key to randomize bucket select with + uint256 nKey; + + //! Serialization versions. + enum Format : uint8_t { + V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 + V1_DETERMINISTIC = 1, //!< for pre-asmap files + V2_ASMAP = 2, //!< for files including asmap version + V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format + V4_MULTIPORT = 4, //!< adds support for multiple ports per IP + }; + + //! The maximum format this software knows it can unserialize. Also, we always serialize + //! in this format. + //! The format (first byte in the serialized stream) can be higher than this and + //! still this software may be able to unserialize the file - if the second byte + //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. + static constexpr Format FILE_FORMAT = Format::V4_MULTIPORT; + + //! The initial value of a field that is incremented every time an incompatible format + //! change is made (such that old software versions would not be able to parse and + //! understand the new file format). This is 32 because we overtook the "key size" + //! field which was 32 historically. + //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. + static constexpr uint8_t INCOMPATIBILITY_BASE = 32; + + //! last used nId + int nIdCount GUARDED_BY(cs){0}; + + //! table with information about all nIds + std::unordered_map mapInfo GUARDED_BY(cs); + + //! find an nId based on its network address and port. + std::unordered_map mapAddr GUARDED_BY(cs); + + //! randomly-ordered vector of all nIds + //! This is mutable because it is unobservable outside the class, so any + //! changes to it (even in const methods) are also unobservable. + mutable std::vector vRandom GUARDED_BY(cs); + + // number of "tried" entries + int nTried GUARDED_BY(cs){0}; + + //! list of "tried" buckets + int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! number of (unique) "new" entries + int nNew GUARDED_BY(cs){0}; + + //! list of "new" buckets + int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. + NodeSeconds m_last_good GUARDED_BY(cs){1s}; + + //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. + std::set m_tried_collisions; + + /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ + const int32_t m_consistency_check_ratio; + + /** Reference to the netgroup manager. netgroupman must be constructed before addrman and destructed after. */ + const NetGroupManager& m_netgroupman; + + //! Find an entry. + AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Create a new entry and add it to the internal data structures mapInfo, mapAddr and vRandom. + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Swap two elements in vRandom. + void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Delete an entry. It must not be in tried, and have refcount 0. + void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Clear a position in a "new" table. This is the only place where entries are actually deleted. + void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Move an entry from the "new" table(s) to the "tried" table + void MakeTried(AddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + /** Attempt to add a single address to addrman's new table. + * @see AddrMan::Add() for parameters. */ + bool AddSingle(const CAddress& addr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + + bool Good_(const CService& addr, bool test_before_evict, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); + + bool Add_(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Attempt_(const CService& addr, bool fCountFailure, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::vector GetAddr_(size_t max_addresses, size_t max_pct, std::optional network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Connected_(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::optional FindAddressEntry_(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Consistency check, taking into account m_consistency_check_ratio. + //! Will std::abort if an inconsistency is detected. + void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Perform consistency check, regardless of m_consistency_check_ratio. + //! @returns an error code or zero. + int CheckAddrman() const EXCLUSIVE_LOCKS_REQUIRED(cs); +}; + +#endif // BITCOIN_ADDRMAN_IMPL_H diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index 0bebb0cf5494..e614102de3f9 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -96,7 +96,7 @@ base_uint& base_uint::operator/=(const base_uint& b) while (shift >= 0) { if (num >= div) { num -= div; - pn[shift / 32] |= (1 << (shift & 31)); // set a bit of the result. + pn[shift / 32] |= (1U << (shift & 31)); // set a bit of the result. } div >>= 1; // shift back. shift--; @@ -146,13 +146,21 @@ double base_uint::getdouble() const template std::string base_uint::GetHex() const { - return ArithToUint256(*this).GetHex(); + base_blob b; + for (int x = 0; x < this->WIDTH; ++x) { + WriteLE32(b.begin() + x*4, this->pn[x]); + } + return b.GetHex(); } template void base_uint::SetHex(const char* psz) { - *this = UintToArith256(uint256S(psz)); + base_blob b; + b.SetHex(psz); + for (int x = 0; x < this->WIDTH; ++x) { + this->pn[x] = ReadLE32(b.begin() + x*4); + } } template @@ -164,7 +172,7 @@ void base_uint::SetHex(const std::string& str) template std::string base_uint::ToString() const { - return (GetHex()); + return GetHex(); } template @@ -183,20 +191,7 @@ unsigned int base_uint::bits() const } // Explicit instantiations for base_uint<256> -template base_uint<256>::base_uint(const std::string&); -template base_uint<256>& base_uint<256>::operator<<=(unsigned int); -template base_uint<256>& base_uint<256>::operator>>=(unsigned int); -template base_uint<256>& base_uint<256>::operator*=(uint32_t b32); -template base_uint<256>& base_uint<256>::operator*=(const base_uint<256>& b); -template base_uint<256>& base_uint<256>::operator/=(const base_uint<256>& b); -template int base_uint<256>::CompareTo(const base_uint<256>&) const; -template bool base_uint<256>::EqualTo(uint64_t) const; -template double base_uint<256>::getdouble() const; -template std::string base_uint<256>::GetHex() const; -template std::string base_uint<256>::ToString() const; -template void base_uint<256>::SetHex(const char*); -template void base_uint<256>::SetHex(const std::string&); -template unsigned int base_uint<256>::bits() const; +template class base_uint<256>; // This implementation directly uses shifts instead of going // through an intermediate MPI representation. @@ -236,7 +231,7 @@ uint32_t arith_uint256::GetCompact(bool fNegative) const nCompact >>= 8; nSize++; } - assert((nCompact & ~0x007fffff) == 0); + assert((nCompact & ~0x007fffffU) == 0); assert(nSize < 256); nCompact |= nSize << 24; nCompact |= (fNegative && (nCompact & 0x007fffff) ? 0x00800000 : 0); diff --git a/src/arith_uint256.h b/src/arith_uint256.h index a0a0429c2a3b..b7b3b3a2852a 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -24,22 +24,19 @@ template class base_uint { protected: + static_assert(BITS / 32 > 0 && BITS % 32 == 0, "Template parameter BITS must be a positive multiple of 32."); static constexpr int WIDTH = BITS / 32; uint32_t pn[WIDTH]; public: base_uint() { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - for (int i = 0; i < WIDTH; i++) pn[i] = 0; } base_uint(const base_uint& b) { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - for (int i = 0; i < WIDTH; i++) pn[i] = b.pn[i]; } @@ -53,8 +50,6 @@ class base_uint base_uint(uint64_t b) { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - pn[0] = (unsigned int)b; pn[1] = (unsigned int)(b >> 32); for (int i = 2; i < WIDTH; i++) @@ -284,4 +279,6 @@ class arith_uint256 : public base_uint<256> { uint256 ArithToUint256(const arith_uint256 &); arith_uint256 UintToArith256(const uint256 &); +extern template class base_uint<256>; + #endif // BITCOIN_ARITH_UINT256_H diff --git a/src/banman.cpp b/src/banman.cpp index d2437e67338c..3cd646c148da 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -1,12 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include -#include +#include +#include #include #include #include @@ -15,44 +16,55 @@ BanMan::BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t default_ban_time) : m_client_interface(client_interface), m_ban_db(std::move(ban_file)), m_default_ban_time(default_ban_time) { + LoadBanlist(); + DumpBanlist(); +} + +BanMan::~BanMan() +{ + DumpBanlist(); +} + +void BanMan::LoadBanlist() +{ + LOCK(m_cs_banned); + if (m_client_interface) m_client_interface->InitMessage(_("Loading banlist…").translated); - int64_t n_start = GetTimeMillis(); - if (m_ban_db.Read(m_banned, m_is_dirty)) { + const auto start{SteadyClock::now()}; + if (m_ban_db.Read(m_banned)) { SweepBanned(); // sweep out unused entries LogPrint(BCLog::NET, "Loaded %d banned node addresses/subnets %dms\n", m_banned.size(), - GetTimeMillis() - n_start); + Ticks(SteadyClock::now() - start)); } else { LogPrintf("Recreating the banlist database\n"); m_banned = {}; m_is_dirty = true; } - - DumpBanlist(); -} - -BanMan::~BanMan() -{ - DumpBanlist(); } void BanMan::DumpBanlist() { - SweepBanned(); // clean unused entries (if bantime has expired) - - if (!BannedSetIsDirty()) return; - - int64_t n_start = GetTimeMillis(); + static Mutex dump_mutex; + LOCK(dump_mutex); banmap_t banmap; - GetBanned(banmap); - if (m_ban_db.Write(banmap)) { + { + LOCK(m_cs_banned); + SweepBanned(); + if (!BannedSetIsDirty()) return; + banmap = m_banned; SetBannedSetDirty(false); } + const auto start{SteadyClock::now()}; + if (!m_ban_db.Write(banmap)) { + SetBannedSetDirty(true); + } + LogPrint(BCLog::NET, "Flushed %d banned node addresses/subnets to disk %dms\n", banmap.size(), - GetTimeMillis() - n_start); + Ticks(SteadyClock::now() - start)); } void BanMan::ClearBanned() @@ -167,23 +179,24 @@ void BanMan::GetBanned(banmap_t& banmap) void BanMan::SweepBanned() { + AssertLockHeld(m_cs_banned); + int64_t now = GetTime(); bool notify_ui = false; - { - LOCK(m_cs_banned); - banmap_t::iterator it = m_banned.begin(); - while (it != m_banned.end()) { - CSubNet sub_net = (*it).first; - CBanEntry ban_entry = (*it).second; - if (!sub_net.IsValid() || now > ban_entry.nBanUntil) { - m_banned.erase(it++); - m_is_dirty = true; - notify_ui = true; - LogPrint(BCLog::NET, "Removed banned node address/subnet: %s\n", sub_net.ToString()); - } else - ++it; + banmap_t::iterator it = m_banned.begin(); + while (it != m_banned.end()) { + CSubNet sub_net = (*it).first; + CBanEntry ban_entry = (*it).second; + if (!sub_net.IsValid() || now > ban_entry.nBanUntil) { + m_banned.erase(it++); + m_is_dirty = true; + notify_ui = true; + LogPrint(BCLog::NET, "Removed banned node address/subnet: %s\n", sub_net.ToString()); + } else { + ++it; } } + // update UI if (notify_ui && m_client_interface) { m_client_interface->BannedListChanged(); diff --git a/src/banman.h b/src/banman.h index 8c75d4037e5f..77b043f081b5 100644 --- a/src/banman.h +++ b/src/banman.h @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BANMAN_H #define BITCOIN_BANMAN_H #include -#include +#include #include #include // For banmap_t #include @@ -80,19 +80,20 @@ class BanMan void DumpBanlist(); private: + void LoadBanlist() EXCLUSIVE_LOCKS_REQUIRED(!m_cs_banned); bool BannedSetIsDirty(); //!set the "dirty" flag for the banlist void SetBannedSetDirty(bool dirty = true); //!clean unused entries (if bantime has expired) - void SweepBanned(); + void SweepBanned() EXCLUSIVE_LOCKS_REQUIRED(m_cs_banned); RecursiveMutex m_cs_banned; banmap_t m_banned GUARDED_BY(m_cs_banned); - bool m_is_dirty GUARDED_BY(m_cs_banned); + bool m_is_dirty GUARDED_BY(m_cs_banned){false}; CClientUIInterface* m_client_interface = nullptr; CBanDB m_ban_db; const int64_t m_default_ban_time; CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; }; -#endif +#endif // BITCOIN_BANMAN_H diff --git a/src/base58.cpp b/src/base58.cpp index fb04673c5c8b..11c1ce7397f4 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2020 The Bitcoin Core developers +// Copyright (c) 2014-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -126,7 +126,7 @@ std::string EncodeBase58(Span input) bool DecodeBase58(const std::string& str, std::vector& vchRet, int max_ret_len) { - if (!ValidAsCString(str)) { + if (!ContainsNoNUL(str)) { return false; } return DecodeBase58(str.c_str(), vchRet, max_ret_len); @@ -149,7 +149,7 @@ std::string EncodeBase58Check(Span input) return false; } // re-calculate the checksum, ensure it matches the included 4-byte checksum - uint256 hash = Hash(MakeSpan(vchRet).first(vchRet.size() - 4)); + uint256 hash = Hash(Span{vchRet}.first(vchRet.size() - 4)); if (memcmp(&hash, &vchRet[vchRet.size() - 4], 4) != 0) { vchRet.clear(); return false; @@ -160,7 +160,7 @@ std::string EncodeBase58Check(Span input) bool DecodeBase58Check(const std::string& str, std::vector& vchRet, int max_ret) { - if (!ValidAsCString(str)) { + if (!ContainsNoNUL(str)) { return false; } return DecodeBase58Check(str.c_str(), vchRet, max_ret); diff --git a/src/base58.h b/src/base58.h index 9ba5af73e059..d2a8d5e3bc47 100644 --- a/src/base58.h +++ b/src/base58.h @@ -14,7 +14,6 @@ #ifndef BITCOIN_BASE58_H #define BITCOIN_BASE58_H -#include #include #include diff --git a/src/bech32.cpp b/src/bech32.cpp index 288b14e02389..8e0025b8f42b 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -1,11 +1,15 @@ // Copyright (c) 2017, 2021 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include +#include #include +#include +#include namespace bech32 { @@ -30,6 +34,90 @@ const int8_t CHARSET_REV[128] = { 1, 0, 3, 16, 11, 28, 12, 14, 6, 4, 2, -1, -1, -1, -1, -1 }; +/** We work with the finite field GF(1024) defined as a degree 2 extension of the base field GF(32) + * The defining polynomial of the extension is x^2 + 9x + 23. + * Let (e) be a root of this defining polynomial. Then (e) is a primitive element of GF(1024), + * that is, a generator of the field. Every non-zero element of the field can then be represented + * as (e)^k for some power k. + * The array GF1024_EXP contains all these powers of (e) - GF1024_EXP[k] = (e)^k in GF(1024). + * Conversely, GF1024_LOG contains the discrete logarithms of these powers, so + * GF1024_LOG[GF1024_EXP[k]] == k. + * The following function generates the two tables GF1024_EXP and GF1024_LOG as constexprs. */ +constexpr std::pair, std::array> GenerateGFTables() +{ + // Build table for GF(32). + // We use these tables to perform arithmetic in GF(32) below, when constructing the + // tables for GF(1024). + std::array GF32_EXP{}; + std::array GF32_LOG{}; + + // fmod encodes the defining polynomial of GF(32) over GF(2), x^5 + x^3 + 1. + // Because coefficients in GF(2) are binary digits, the coefficients are packed as 101001. + const int fmod = 41; + + // Elements of GF(32) are encoded as vectors of length 5 over GF(2), that is, + // 5 binary digits. Each element (b_4, b_3, b_2, b_1, b_0) encodes a polynomial + // b_4*x^4 + b_3*x^3 + b_2*x^2 + b_1*x^1 + b_0 (modulo fmod). + // For example, 00001 = 1 is the multiplicative identity. + GF32_EXP[0] = 1; + GF32_LOG[0] = -1; + GF32_LOG[1] = 0; + int v = 1; + for (int i = 1; i < 31; ++i) { + // Multiplication by x is the same as shifting left by 1, as + // every coefficient of the polynomial is moved up one place. + v = v << 1; + // If the polynomial now has an x^5 term, we subtract fmod from it + // to remain working modulo fmod. Subtraction is the same as XOR in characteristic + // 2 fields. + if (v & 32) v ^= fmod; + GF32_EXP[i] = v; + GF32_LOG[v] = i; + } + + // Build table for GF(1024) + std::array GF1024_EXP{}; + std::array GF1024_LOG{}; + + GF1024_EXP[0] = 1; + GF1024_LOG[0] = -1; + GF1024_LOG[1] = 0; + + // Each element v of GF(1024) is encoded as a 10 bit integer in the following way: + // v = v1 || v0 where v0, v1 are 5-bit integers (elements of GF(32)). + // The element (e) is encoded as 1 || 0, to represent 1*(e) + 0. Every other element + // a*(e) + b is represented as a || b (a and b are both GF(32) elements). Given (v), + // we compute (e)*(v) by multiplying in the following way: + // + // v0' = 23*v1 + // v1' = 9*v1 + v0 + // e*v = v1' || v0' + // + // Where 23, 9 are GF(32) elements encoded as described above. Multiplication in GF(32) + // is done using the log/exp tables: + // e^x * e^y = e^(x + y) so a * b = EXP[ LOG[a] + LOG [b] ] + // for non-zero a and b. + + v = 1; + for (int i = 1; i < 1023; ++i) { + int v0 = v & 31; + int v1 = v >> 5; + + int v0n = v1 ? GF32_EXP.at((GF32_LOG.at(v1) + GF32_LOG.at(23)) % 31) : 0; + int v1n = (v1 ? GF32_EXP.at((GF32_LOG.at(v1) + GF32_LOG.at(9)) % 31) : 0) ^ v0; + + v = v1n << 5 | v0n; + GF1024_EXP[i] = v; + GF1024_LOG[v] = i; + } + + return std::make_pair(GF1024_EXP, GF1024_LOG); +} + +constexpr auto tables = GenerateGFTables(); +constexpr const std::array& GF1024_EXP = tables.first; +constexpr const std::array& GF1024_LOG = tables.second; + /* Determine the final constant to use for the specified encoding. */ uint32_t EncodingConstant(Encoding encoding) { assert(encoding == Encoding::BECH32 || encoding == Encoding::BECH32M); @@ -66,6 +154,26 @@ uint32_t PolyMod(const data& v) // the above example, `c` initially corresponds to 1 mod g(x), and after processing 2 inputs of // v, it corresponds to x^2 + v0*x + v1 mod g(x). As 1 mod g(x) = 1, that is the starting value // for `c`. + + // The following Sage code constructs the generator used: + // + // B = GF(2) # Binary field + // BP. = B[] # Polynomials over the binary field + // F_mod = b**5 + b**3 + 1 + // F. = GF(32, modulus=F_mod, repr='int') # GF(32) definition + // FP. = F[] # Polynomials over GF(32) + // E_mod = x**2 + F.fetch_int(9)*x + F.fetch_int(23) + // E. = F.extension(E_mod) # GF(1024) extension field definition + // for p in divisors(E.order() - 1): # Verify e has order 1023. + // assert((e**p == 1) == (p % 1023 == 0)) + // G = lcm([(e**i).minpoly() for i in range(997,1000)]) + // print(G) # Print out the generator + // + // It demonstrates that g(x) is the least common multiple of the minimal polynomials + // of 3 consecutive powers (997,998,999) of a primitive element (e) of GF(1024). + // That guarantees it is, in fact, the generator of a primitive BCH code with cycle + // length 1023 and distance 4. See https://en.wikipedia.org/wiki/BCH_code for more details. + uint32_t c = 1; for (const auto v_i : v) { // We want to update `c` to correspond to a polynomial with one extra term. If the initial @@ -88,22 +196,118 @@ uint32_t PolyMod(const data& v) // Then compute c1*x^5 + c2*x^4 + c3*x^3 + c4*x^2 + c5*x + v_i: c = ((c & 0x1ffffff) << 5) ^ v_i; - // Finally, for each set bit n in c0, conditionally add {2^n}k(x): + // Finally, for each set bit n in c0, conditionally add {2^n}k(x). These constants can be + // computed using the following Sage code (continuing the code above): + // + // for i in [1,2,4,8,16]: # Print out {1,2,4,8,16}*(g(x) mod x^6), packed in hex integers. + // v = 0 + // for coef in reversed((F.fetch_int(i)*(G % x**6)).coefficients(sparse=True)): + // v = v*32 + coef.integer_representation() + // print("0x%x" % v) + // if (c0 & 1) c ^= 0x3b6a57b2; // k(x) = {29}x^5 + {22}x^4 + {20}x^3 + {21}x^2 + {29}x + {18} if (c0 & 2) c ^= 0x26508e6d; // {2}k(x) = {19}x^5 + {5}x^4 + x^3 + {3}x^2 + {19}x + {13} if (c0 & 4) c ^= 0x1ea119fa; // {4}k(x) = {15}x^5 + {10}x^4 + {2}x^3 + {6}x^2 + {15}x + {26} if (c0 & 8) c ^= 0x3d4233dd; // {8}k(x) = {30}x^5 + {20}x^4 + {4}x^3 + {12}x^2 + {30}x + {29} if (c0 & 16) c ^= 0x2a1462b3; // {16}k(x) = {21}x^5 + x^4 + {8}x^3 + {24}x^2 + {21}x + {19} + } return c; } +/** Syndrome computes the values s_j = R(e^j) for j in [997, 998, 999]. As described above, the + * generator polynomial G is the LCM of the minimal polynomials of (e)^997, (e)^998, and (e)^999. + * + * Consider a codeword with errors, of the form R(x) = C(x) + E(x). The residue is the bit-packed + * result of computing R(x) mod G(X), where G is the generator of the code. Because C(x) is a valid + * codeword, it is a multiple of G(X), so the residue is in fact just E(x) mod G(x). Note that all + * of the (e)^j are roots of G(x) by definition, so R((e)^j) = E((e)^j). + * + * Let R(x) = r1*x^5 + r2*x^4 + r3*x^3 + r4*x^2 + r5*x + r6 + * + * To compute R((e)^j), we are really computing: + * r1*(e)^(j*5) + r2*(e)^(j*4) + r3*(e)^(j*3) + r4*(e)^(j*2) + r5*(e)^j + r6 + * + * Now note that all of the (e)^(j*i) for i in [5..0] are constants and can be precomputed. + * But even more than that, we can consider each coefficient as a bit-string. + * For example, take r5 = (b_5, b_4, b_3, b_2, b_1) written out as 5 bits. Then: + * r5*(e)^j = b_1*(e)^j + b_2*(2*(e)^j) + b_3*(4*(e)^j) + b_4*(8*(e)^j) + b_5*(16*(e)^j) + * where all the (2^i*(e)^j) are constants and can be precomputed. + * + * Then we just add each of these corresponding constants to our final value based on the + * bit values b_i. This is exactly what is done in the Syndrome function below. + */ +constexpr std::array GenerateSyndromeConstants() { + std::array SYNDROME_CONSTS{}; + for (int k = 1; k < 6; ++k) { + for (int shift = 0; shift < 5; ++shift) { + int16_t b = GF1024_LOG.at(size_t{1} << shift); + int16_t c0 = GF1024_EXP.at((997*k + b) % 1023); + int16_t c1 = GF1024_EXP.at((998*k + b) % 1023); + int16_t c2 = GF1024_EXP.at((999*k + b) % 1023); + uint32_t c = c2 << 20 | c1 << 10 | c0; + int ind = 5*(k-1) + shift; + SYNDROME_CONSTS[ind] = c; + } + } + return SYNDROME_CONSTS; +} +constexpr std::array SYNDROME_CONSTS = GenerateSyndromeConstants(); + +/** + * Syndrome returns the three values s_997, s_998, and s_999 described above, + * packed into a 30-bit integer, where each group of 10 bits encodes one value. + */ +uint32_t Syndrome(const uint32_t residue) { + // low is the first 5 bits, corresponding to the r6 in the residue + // (the constant term of the polynomial). + uint32_t low = residue & 0x1f; + + // We begin by setting s_j = low = r6 for all three values of j, because these are unconditional. + uint32_t result = low ^ (low << 10) ^ (low << 20); + + // Then for each following bit, we add the corresponding precomputed constant if the bit is 1. + // For example, 0x31edd3c4 is 1100011110 1101110100 1111000100 when unpacked in groups of 10 + // bits, corresponding exactly to a^999 || a^998 || a^997 (matching the corresponding values in + // GF1024_EXP above). In this way, we compute all three values of s_j for j in (997, 998, 999) + // simultaneously. Recall that XOR corresponds to addition in a characteristic 2 field. + for (int i = 0; i < 25; ++i) { + result ^= ((residue >> (5+i)) & 1 ? SYNDROME_CONSTS.at(i) : 0); + } + return result; +} + /** Convert to lower case. */ inline unsigned char LowerCase(unsigned char c) { return (c >= 'A' && c <= 'Z') ? (c - 'A') + 'a' : c; } +/** Return indices of invalid characters in a Bech32 string. */ +bool CheckCharacters(const std::string& str, std::vector& errors) +{ + bool lower = false, upper = false; + for (size_t i = 0; i < str.size(); ++i) { + unsigned char c{(unsigned char)(str[i])}; + if (c >= 'a' && c <= 'z') { + if (upper) { + errors.push_back(i); + } else { + lower = true; + } + } else if (c >= 'A' && c <= 'Z') { + if (lower) { + errors.push_back(i); + } else { + upper = true; + } + } else if (c < 33 || c > 126) { + errors.push_back(i); + } + } + return errors.empty(); +} + /** Expand a HRP for use in checksum computation. */ data ExpandHRP(const std::string& hrp) { @@ -125,7 +329,8 @@ Encoding VerifyChecksum(const std::string& hrp, const data& values) // PolyMod computes what value to xor into the final values to make the checksum 0. However, // if we required that the checksum was 0, it would be the case that appending a 0 to a valid // list of values would result in a new valid list. For that reason, Bech32 requires the - // resulting checksum to be 1 instead. In Bech32m, this constant was amended. + // resulting checksum to be 1 instead. In Bech32m, this constant was amended. See + // https://gist.github.com/sipa/14c248c288c3880a3b191f978a34508e for details. const uint32_t check = PolyMod(Cat(ExpandHRP(hrp), values)); if (check == EncodingConstant(Encoding::BECH32)) return Encoding::BECH32; if (check == EncodingConstant(Encoding::BECH32M)) return Encoding::BECH32M; @@ -166,14 +371,8 @@ std::string Encode(Encoding encoding, const std::string& hrp, const data& values /** Decode a Bech32 or Bech32m string. */ DecodeResult Decode(const std::string& str) { - bool lower = false, upper = false; - for (size_t i = 0; i < str.size(); ++i) { - unsigned char c = str[i]; - if (c >= 'a' && c <= 'z') lower = true; - else if (c >= 'A' && c <= 'Z') upper = true; - else if (c < 33 || c > 126) return {}; - } - if (lower && upper) return {}; + std::vector errors; + if (!CheckCharacters(str, errors)) return {}; size_t pos = str.rfind('1'); if (str.size() > 90 || pos == str.npos || pos == 0 || pos + 7 > str.size()) { return {}; @@ -197,4 +396,174 @@ DecodeResult Decode(const std::string& str) { return {result, std::move(hrp), data(values.begin(), values.end() - 6)}; } +/** Find index of an incorrect character in a Bech32 string. */ +std::pair> LocateErrors(const std::string& str) { + std::vector error_locations{}; + + if (str.size() > 90) { + error_locations.resize(str.size() - 90); + std::iota(error_locations.begin(), error_locations.end(), 90); + return std::make_pair("Bech32 string too long", std::move(error_locations)); + } + + if (!CheckCharacters(str, error_locations)){ + return std::make_pair("Invalid character or mixed case", std::move(error_locations)); + } + + size_t pos = str.rfind('1'); + if (pos == str.npos) { + return std::make_pair("Missing separator", std::vector{}); + } + if (pos == 0 || pos + 7 > str.size()) { + error_locations.push_back(pos); + return std::make_pair("Invalid separator position", std::move(error_locations)); + } + + std::string hrp; + for (size_t i = 0; i < pos; ++i) { + hrp += LowerCase(str[i]); + } + + size_t length = str.size() - 1 - pos; // length of data part + data values(length); + for (size_t i = pos + 1; i < str.size(); ++i) { + unsigned char c = str[i]; + int8_t rev = CHARSET_REV[c]; + if (rev == -1) { + error_locations.push_back(i); + return std::make_pair("Invalid Base 32 character", std::move(error_locations)); + } + values[i - pos - 1] = rev; + } + + // We attempt error detection with both bech32 and bech32m, and choose the one with the fewest errors + // We can't simply use the segwit version, because that may be one of the errors + std::optional error_encoding; + for (Encoding encoding : {Encoding::BECH32, Encoding::BECH32M}) { + std::vector possible_errors; + // Recall that (ExpandHRP(hrp) ++ values) is interpreted as a list of coefficients of a polynomial + // over GF(32). PolyMod computes the "remainder" of this polynomial modulo the generator G(x). + uint32_t residue = PolyMod(Cat(ExpandHRP(hrp), values)) ^ EncodingConstant(encoding); + + // All valid codewords should be multiples of G(x), so this remainder (after XORing with the encoding + // constant) should be 0 - hence 0 indicates there are no errors present. + if (residue != 0) { + // If errors are present, our polynomial must be of the form C(x) + E(x) where C is the valid + // codeword (a multiple of G(x)), and E encodes the errors. + uint32_t syn = Syndrome(residue); + + // Unpack the three 10-bit syndrome values + int s0 = syn & 0x3FF; + int s1 = (syn >> 10) & 0x3FF; + int s2 = syn >> 20; + + // Get the discrete logs of these values in GF1024 for more efficient computation + int l_s0 = GF1024_LOG.at(s0); + int l_s1 = GF1024_LOG.at(s1); + int l_s2 = GF1024_LOG.at(s2); + + // First, suppose there is only a single error. Then E(x) = e1*x^p1 for some position p1 + // Then s0 = E((e)^997) = e1*(e)^(997*p1) and s1 = E((e)^998) = e1*(e)^(998*p1) + // Therefore s1/s0 = (e)^p1, and by the same logic, s2/s1 = (e)^p1 too. + // Hence, s1^2 == s0*s2, which is exactly the condition we check first: + if (l_s0 != -1 && l_s1 != -1 && l_s2 != -1 && (2 * l_s1 - l_s2 - l_s0 + 2046) % 1023 == 0) { + // Compute the error position p1 as l_s1 - l_s0 = p1 (mod 1023) + size_t p1 = (l_s1 - l_s0 + 1023) % 1023; // the +1023 ensures it is positive + // Now because s0 = e1*(e)^(997*p1), we get e1 = s0/((e)^(997*p1)). Remember that (e)^1023 = 1, + // so 1/((e)^997) = (e)^(1023-997). + int l_e1 = l_s0 + (1023 - 997) * p1; + // Finally, some sanity checks on the result: + // - The error position should be within the length of the data + // - e1 should be in GF(32), which implies that e1 = (e)^(33k) for some k (the 31 non-zero elements + // of GF(32) form an index 33 subgroup of the 1023 non-zero elements of GF(1024)). + if (p1 < length && !(l_e1 % 33)) { + // Polynomials run from highest power to lowest, so the index p1 is from the right. + // We don't return e1 because it is dangerous to suggest corrections to the user, + // the user should check the address themselves. + possible_errors.push_back(str.size() - p1 - 1); + } + // Otherwise, suppose there are two errors. Then E(x) = e1*x^p1 + e2*x^p2. + } else { + // For all possible first error positions p1 + for (size_t p1 = 0; p1 < length; ++p1) { + // We have guessed p1, and want to solve for p2. Recall that E(x) = e1*x^p1 + e2*x^p2, so + // s0 = E((e)^997) = e1*(e)^(997^p1) + e2*(e)^(997*p2), and similar for s1 and s2. + // + // Consider s2 + s1*(e)^p1 + // = 2e1*(e)^(999^p1) + e2*(e)^(999*p2) + e2*(e)^(998*p2)*(e)^p1 + // = e2*(e)^(999*p2) + e2*(e)^(998*p2)*(e)^p1 + // (Because we are working in characteristic 2.) + // = e2*(e)^(998*p2) ((e)^p2 + (e)^p1) + // + int s2_s1p1 = s2 ^ (s1 == 0 ? 0 : GF1024_EXP.at((l_s1 + p1) % 1023)); + if (s2_s1p1 == 0) continue; + int l_s2_s1p1 = GF1024_LOG.at(s2_s1p1); + + // Similarly, s1 + s0*(e)^p1 + // = e2*(e)^(997*p2) ((e)^p2 + (e)^p1) + int s1_s0p1 = s1 ^ (s0 == 0 ? 0 : GF1024_EXP.at((l_s0 + p1) % 1023)); + if (s1_s0p1 == 0) continue; + int l_s1_s0p1 = GF1024_LOG.at(s1_s0p1); + + // So, putting these together, we can compute the second error position as + // (e)^p2 = (s2 + s1^p1)/(s1 + s0^p1) + // p2 = log((e)^p2) + size_t p2 = (l_s2_s1p1 - l_s1_s0p1 + 1023) % 1023; + + // Sanity checks that p2 is a valid position and not the same as p1 + if (p2 >= length || p1 == p2) continue; + + // Now we want to compute the error values e1 and e2. + // Similar to above, we compute s1 + s0*(e)^p2 + // = e1*(e)^(997*p1) ((e)^p1 + (e)^p2) + int s1_s0p2 = s1 ^ (s0 == 0 ? 0 : GF1024_EXP.at((l_s0 + p2) % 1023)); + if (s1_s0p2 == 0) continue; + int l_s1_s0p2 = GF1024_LOG.at(s1_s0p2); + + // And compute (the log of) 1/((e)^p1 + (e)^p2)) + int inv_p1_p2 = 1023 - GF1024_LOG.at(GF1024_EXP.at(p1) ^ GF1024_EXP.at(p2)); + + // Then (s1 + s0*(e)^p1) * (1/((e)^p1 + (e)^p2))) + // = e2*(e)^(997*p2) + // Then recover e2 by dividing by (e)^(997*p2) + int l_e2 = l_s1_s0p1 + inv_p1_p2 + (1023 - 997) * p2; + // Check that e2 is in GF(32) + if (l_e2 % 33) continue; + + // In the same way, (s1 + s0*(e)^p2) * (1/((e)^p1 + (e)^p2))) + // = e1*(e)^(997*p1) + // So recover e1 by dividing by (e)^(997*p1) + int l_e1 = l_s1_s0p2 + inv_p1_p2 + (1023 - 997) * p1; + // Check that e1 is in GF(32) + if (l_e1 % 33) continue; + + // Again, we do not return e1 or e2 for safety. + // Order the error positions from the left of the string and return them + if (p1 > p2) { + possible_errors.push_back(str.size() - p1 - 1); + possible_errors.push_back(str.size() - p2 - 1); + } else { + possible_errors.push_back(str.size() - p2 - 1); + possible_errors.push_back(str.size() - p1 - 1); + } + break; + } + } + } else { + // No errors + return std::make_pair("", std::vector{}); + } + + if (error_locations.empty() || (!possible_errors.empty() && possible_errors.size() < error_locations.size())) { + error_locations = std::move(possible_errors); + if (!error_locations.empty()) error_encoding = encoding; + } + } + std::string error_message = error_encoding == Encoding::BECH32M ? "Invalid Bech32m checksum" + : error_encoding == Encoding::BECH32 ? "Invalid Bech32 checksum" + : "Invalid checksum"; + + return std::make_pair(error_message, std::move(error_locations)); +} + } // namespace bech32 diff --git a/src/bech32.h b/src/bech32.h index e9450ccc2b35..5e89e6efdaa9 100644 --- a/src/bech32.h +++ b/src/bech32.h @@ -1,4 +1,5 @@ // Copyright (c) 2017, 2021 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -44,6 +45,9 @@ struct DecodeResult /** Decode a Bech32 or Bech32m string. */ DecodeResult Decode(const std::string& str); +/** Return the positions of errors in a Bech32 string. */ +std::pair> LocateErrors(const std::string& str); + } // namespace bech32 #endif // BITCOIN_BECH32_H diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index b7bd8a32612b..019b1333458f 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -1,10 +1,12 @@ -// Copyright (c) 2020-2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include +#include #include +#include #include #include @@ -15,6 +17,9 @@ static constexpr size_t NUM_SOURCES = 64; static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; +static NetGroupManager EMPTY_NETGROUPMAN{std::vector()}; +static constexpr uint32_t ADDRMAN_CONSISTENCY_CHECK_RATIO{0}; + static std::vector g_sources; static std::vector> g_addresses; @@ -38,7 +43,7 @@ static void CreateAddresses() CAddress ret(CService(addr, port), NODE_NETWORK); - ret.nTime = GetAdjustedTime(); + ret.nTime = Now(); return ret; }; @@ -52,14 +57,14 @@ static void CreateAddresses() } } -static void AddAddressesToAddrMan(CAddrMan& addrman) +static void AddAddressesToAddrMan(AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { addrman.Add(g_addresses[source_i], g_sources[source_i]); } } -static void FillAddrMan(CAddrMan& addrman) +static void FillAddrMan(AddrMan& addrman) { CreateAddresses(); @@ -72,70 +77,63 @@ static void AddrManAdd(benchmark::Bench& bench) { CreateAddresses(); - CAddrMan addrman; - bench.run([&] { + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); - addrman.Clear(); }); } static void AddrManSelect(benchmark::Bench& bench) { - CAddrMan addrman; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); bench.run([&] { const auto& address = addrman.Select(); - assert(address.GetPort() > 0); + assert(address.first.GetPort() > 0); }); } static void AddrManGetAddr(benchmark::Bench& bench) { - CAddrMan addrman; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); bench.run([&] { - const auto& addresses = addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23, /* network */ std::nullopt); + const auto& addresses = addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt); assert(addresses.size() > 0); }); } -static void AddrManGood(benchmark::Bench& bench) +static void AddrManAddThenGood(benchmark::Bench& bench) { - /* Create many CAddrMan objects - one to be modified at each loop iteration. - * This is necessary because the CAddrMan::Good() method modifies the - * object, affecting the timing of subsequent calls to the same method and - * we want to do the same amount of work in every loop iteration. */ - - bench.epochs(5).epochIterations(1); - - std::vector addrmans(bench.epochs() * bench.epochIterations()); - for (auto& addrman : addrmans) { - FillAddrMan(addrman); - } - - auto markSomeAsGood = [](CAddrMan& addrman) { + auto markSomeAsGood = [](AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { - if (addr_i % 32 == 0) { - addrman.Good(g_addresses[source_i][addr_i]); - } + addrman.Good(g_addresses[source_i][addr_i]); } } }; - uint64_t i = 0; + CreateAddresses(); + bench.run([&] { - markSomeAsGood(addrmans.at(i)); - ++i; + // To make the benchmark independent of the number of evaluations, we always prepare a new addrman. + // This is necessary because AddrMan::Good() method modifies the object, affecting the timing of subsequent calls + // to the same method and we want to do the same amount of work in every loop iteration. + // + // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in + // AddrMan::Good() will still be noticeable. + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddAddressesToAddrMan(addrman); + + markSomeAsGood(addrman); }); } -BENCHMARK(AddrManAdd); -BENCHMARK(AddrManSelect); -BENCHMARK(AddrManGetAddr); -BENCHMARK(AddrManGood); +BENCHMARK(AddrManAdd, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManSelect, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManGetAddr, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManAddThenGood, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 6f6b4e3bfa9c..3d08b7201b71 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -50,6 +50,6 @@ static void Base58Decode(benchmark::Bench& bench) } -BENCHMARK(Base58Encode); -BENCHMARK(Base58CheckEncode); -BENCHMARK(Base58Decode); +BENCHMARK(Base58Encode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Base58CheckEncode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Base58Decode, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 8e10862a37c9..1a166e70812b 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -1,9 +1,8 @@ -// Copyright (c) 2018-2020 The Bitcoin Core developers +// Copyright (c) 2018-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include #include #include @@ -33,5 +32,5 @@ static void Bech32Decode(benchmark::Bench& bench) } -BENCHMARK(Bech32Encode); -BENCHMARK(Bech32Decode); +BENCHMARK(Bech32Encode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Bech32Decode, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 012057e792d2..1a3a0062865c 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,72 +1,125 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include +#include #include -#include +#include +#include +#include +#include +#include +#include #include +#include +#include + +using namespace std::chrono_literals; const std::function G_TEST_LOG_FUN{}; +const std::function()> G_TEST_COMMAND_LINE_ARGUMENTS{}; + namespace { -void GenerateTemplateResults(const std::vector& benchmarkResults, const std::string& filename, const char* tpl) +void GenerateTemplateResults(const std::vector& benchmarkResults, const fs::path& file, const char* tpl) { - if (benchmarkResults.empty() || filename.empty()) { + if (benchmarkResults.empty() || file.empty()) { // nothing to write, bail out return; } - std::ofstream fout(filename); + std::ofstream fout{file}; if (fout.is_open()) { ankerl::nanobench::render(tpl, benchmarkResults, fout); + std::cout << "Created " << file << std::endl; } else { - std::cout << "Could write to file '" << filename << "'" << std::endl; + std::cout << "Could not write to file " << file << std::endl; } - - std::cout << "Created '" << filename << "'" << std::endl; } } // namespace -benchmark::BenchRunner::BenchmarkMap& benchmark::BenchRunner::benchmarks() +namespace benchmark { + +// map a label to one or multiple priority levels +std::map map_label_priority = { + {"high", PriorityLevel::HIGH}, + {"low", PriorityLevel::LOW}, + {"all", 0xff} +}; + +std::string ListPriorities() +{ + using item_t = std::pair; + auto sort_by_priority = [](item_t a, item_t b){ return a.second < b.second; }; + std::set sorted_priorities(map_label_priority.begin(), map_label_priority.end(), sort_by_priority); + return Join(sorted_priorities, ',', [](const auto& entry){ return entry.first; }); +} + +uint8_t StringToPriority(const std::string& str) +{ + auto it = map_label_priority.find(str); + if (it == map_label_priority.end()) throw std::runtime_error(strprintf("Unknown priority level %s", str)); + return it->second; +} + +BenchRunner::BenchmarkMap& BenchRunner::benchmarks() { - static std::map benchmarks_map; + static BenchmarkMap benchmarks_map; return benchmarks_map; } -benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func) +BenchRunner::BenchRunner(std::string name, BenchFunction func, PriorityLevel level) { - benchmarks().insert(std::make_pair(name, func)); + benchmarks().insert(std::make_pair(name, std::make_pair(func, level))); } -void benchmark::BenchRunner::RunAll(const Args& args) +void BenchRunner::RunAll(const Args& args) { std::regex reFilter(args.regex_filter); std::smatch baseMatch; + if (args.sanity_check) { + std::cout << "Running with --sanity-check option, benchmark results will be useless." << std::endl; + } + std::vector benchmarkResults; - for (const auto& p : benchmarks()) { - if (!std::regex_match(p.first, baseMatch, reFilter)) { + for (const auto& [name, bench_func] : benchmarks()) { + const auto& [func, priority_level] = bench_func; + + if (!(priority_level & args.priority)) { + continue; + } + + if (!std::regex_match(name, baseMatch, reFilter)) { continue; } if (args.is_list_only) { - std::cout << p.first << std::endl; + std::cout << name << std::endl; continue; } Bench bench; - bench.name(p.first); + if (args.sanity_check) { + bench.epochs(1).epochIterations(1); + } + bench.name(name); + if (args.min_time > 0ms) { + // convert to nanos before dividing to reduce rounding errors + std::chrono::nanoseconds min_time_ns = args.min_time; + bench.minEpochTime(min_time_ns / bench.epochs()); + } + if (args.asymptote.empty()) { - p.second(bench); + func(bench); } else { for (auto n : args.asymptote) { bench.complexityN(n); - p.second(bench); + func(bench); } std::cout << bench.complexityBigO() << std::endl; } @@ -81,3 +134,5 @@ void benchmark::BenchRunner::RunAll(const Args& args) "{{/result}}"); GenerateTemplateResults(benchmarkResults, args.output_json, ankerl::nanobench::templates::json()); } + +} // namespace benchmark diff --git a/src/bench/bench.h b/src/bench/bench.h index c4fcd80e33b0..63e1bf67e21e 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -1,10 +1,11 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include #include #include @@ -40,28 +41,42 @@ using ankerl::nanobench::Bench; typedef std::function BenchFunction; +enum PriorityLevel : uint8_t +{ + LOW = 1 << 0, + HIGH = 1 << 2, +}; + +// List priority labels, comma-separated and sorted by increasing priority +std::string ListPriorities(); +uint8_t StringToPriority(const std::string& str); + struct Args { - std::string regex_filter; bool is_list_only; + bool sanity_check; + std::chrono::milliseconds min_time; std::vector asymptote; - std::string output_csv; - std::string output_json; + fs::path output_csv; + fs::path output_json; + std::string regex_filter; + uint8_t priority; }; class BenchRunner { - typedef std::map BenchmarkMap; + // maps from "name" -> (function, priority_level) + typedef std::map> BenchmarkMap; static BenchmarkMap& benchmarks(); public: - BenchRunner(std::string name, BenchFunction func); + BenchRunner(std::string name, BenchFunction func, PriorityLevel level); static void RunAll(const Args& args); }; } // namespace benchmark -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); -#define BENCHMARK(n) \ - benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo, priority_level); +#define BENCHMARK(n, priority_level) \ + benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n, priority_level); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index aab777cac152..1ac8db19fd88 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,26 +1,39 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include +#include #include #include -#include +#include +#include +#include +#include +#include static const char* DEFAULT_BENCH_FILTER = ".*"; +static constexpr int64_t DEFAULT_MIN_TIME_MS{10}; +/** Priority level default value, run "all" priority levels */ +static const std::string DEFAULT_PRIORITY{"all"}; static void SetupBenchArgs(ArgsManager& argsman) { SetupHelpOptions(argsman); - argsman.AddArg("-asymptote=n1,n2,n3,...", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-asymptote=", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-filter=", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-output_csv=", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-output_json=", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-min-time=", strprintf("Minimum runtime per benchmark, in milliseconds (default: %d)", DEFAULT_MIN_TIME_MS), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); + argsman.AddArg("-output-csv=", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-output-json=", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-priority-level=", strprintf("Run benchmarks of one or multiple priority level(s) (%s), default: '%s'", + benchmark::ListPriorities(), DEFAULT_PRIORITY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } // parses a comma separated list like "10,20,30,50" @@ -36,6 +49,14 @@ static std::vector parseAsymptote(const std::string& str) { return numbers; } +static uint8_t parsePriorityLevel(const std::string& str) { + uint8_t levels{0}; + for (const auto& level: SplitString(str, ',')) { + levels |= benchmark::StringToPriority(level); + } + return levels; +} + int main(int argc, char** argv) { ArgsManager argsman; @@ -48,19 +69,71 @@ int main(int argc, char** argv) } if (HelpRequested(argsman)) { - std::cout << argsman.GetHelpMessage(); + std::cout << "Usage: bench_bitcoin [options]\n" + "\n" + << argsman.GetHelpMessage() + << "Description:\n" + "\n" + " bench_bitcoin executes microbenchmarks. The quality of the benchmark results\n" + " highly depend on the stability of the machine. It can sometimes be difficult\n" + " to get stable, repeatable results, so here are a few tips:\n" + "\n" + " * Use pyperf [1] to disable frequency scaling, turbo boost etc. For best\n" + " results, use CPU pinning and CPU isolation (see [2]).\n" + "\n" + " * Each call of run() should do exactly the same work. E.g. inserting into\n" + " a std::vector doesn't do that as it will reallocate on certain calls. Make\n" + " sure each run has exactly the same preconditions.\n" + "\n" + " * If results are still not reliable, increase runtime with e.g.\n" + " -min-time=5000 to let a benchmark run for at least 5 seconds.\n" + "\n" + " * bench_bitcoin uses nanobench [3] for which there is extensive\n" + " documentation available online.\n" + "\n" + "Environment Variables:\n" + "\n" + " To attach a profiler you can run a benchmark in endless mode. This can be\n" + " done with the environment variable NANOBENCH_ENDLESS. E.g. like so:\n" + "\n" + " NANOBENCH_ENDLESS=MuHash ./bench_bitcoin -filter=MuHash\n" + "\n" + " In rare cases it can be useful to suppress stability warnings. This can be\n" + " done with the environment variable NANOBENCH_SUPPRESS_WARNINGS, e.g:\n" + "\n" + " NANOBENCH_SUPPRESS_WARNINGS=1 ./bench_bitcoin\n" + "\n" + "Notes:\n" + "\n" + " 1. pyperf\n" + " https://github.com/psf/pyperf\n" + "\n" + " 2. CPU pinning & isolation\n" + " https://pyperf.readthedocs.io/en/latest/system.html\n" + "\n" + " 3. nanobench\n" + " https://github.com/martinus/nanobench\n" + "\n"; return EXIT_SUCCESS; } - benchmark::Args args; - args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); - args.is_list_only = argsman.GetBoolArg("-list", false); - args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); - args.output_csv = argsman.GetArg("-output_csv", ""); - args.output_json = argsman.GetArg("-output_json", ""); + try { + benchmark::Args args; + args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); + args.is_list_only = argsman.GetBoolArg("-list", false); + args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS)); + args.output_csv = argsman.GetPathArg("-output-csv"); + args.output_json = argsman.GetPathArg("-output-json"); + args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); + args.sanity_check = argsman.GetBoolArg("-sanity-check", false); + args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY)); - benchmark::BenchRunner::RunAll(args); + benchmark::BenchRunner::RunAll(args); - return EXIT_SUCCESS; + return EXIT_SUCCESS; + } catch (const std::exception& e) { + tfm::format(std::cerr, "Error: %s\n", e.what()); + return EXIT_FAILURE; + } } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index b4b33d115f31..69258377d5be 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2020 The Bitcoin Core developers +// Copyright (c) 2011-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -34,10 +33,10 @@ static void AssembleBlock(benchmark::Bench& bench) txs.at(b) = MakeTransactionRef(tx); } { - LOCK(::cs_main); // Required for ::AcceptToMemoryPool. + LOCK(::cs_main); for (const auto& txr : txs) { - const MempoolAcceptResult res = ::AcceptToMemoryPool(test_setup->m_node.chainman->ActiveChainstate(), *test_setup->m_node.mempool, txr, false /* bypass_limits */); + const MempoolAcceptResult res = test_setup->m_node.chainman->ProcessTransaction(txr); assert(res.m_result_type == MempoolAcceptResult::ResultType::VALID); } } @@ -47,4 +46,4 @@ static void AssembleBlock(benchmark::Bench& bench) }); } -BENCHMARK(AssembleBlock); +BENCHMARK(AssembleBlock, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index d5275b0b763f..5d55ed933209 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -45,10 +45,10 @@ static void CCoinsCaching(benchmark::Bench& bench) // Benchmark. const CTransaction tx_1(t1); bench.run([&] { - bool success = AreInputsStandard(tx_1, coins, false); + bool success{AreInputsStandard(tx_1, coins)}; assert(success); }); ECC_Stop(); } -BENCHMARK(CCoinsCaching); +BENCHMARK(CCoinsCaching, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index a6f4eec4ca9e..9584dd58bb40 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -39,6 +39,6 @@ static void CHACHA20_1MB(benchmark::Bench& bench) CHACHA20(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(CHACHA20_64BYTES); -BENCHMARK(CHACHA20_256BYTES); -BENCHMARK(CHACHA20_1MB); +BENCHMARK(CHACHA20_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index e994279a4d8b..15b3a4f31061 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -115,12 +115,12 @@ static void HASH_1MB(benchmark::Bench& bench) HASH(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT); -BENCHMARK(HASH_64BYTES); -BENCHMARK(HASH_256BYTES); -BENCHMARK(HASH_1MB); +BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index a9f3f5f84d20..747279c1614b 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include // These are the two major time-sinks which happen after we have fully received @@ -17,8 +18,8 @@ static void DeserializeBlockTest(benchmark::Bench& bench) { CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); - char a = '\0'; - stream.write(&a, 1); // Prevent compaction + std::byte a{0}; + stream.write({&a, 1}); // Prevent compaction bench.unit("block").run([&] { CBlock block; @@ -31,8 +32,8 @@ static void DeserializeBlockTest(benchmark::Bench& bench) static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) { CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); - char a = '\0'; - stream.write(&a, 1); // Prevent compaction + std::byte a{0}; + stream.write({&a, 1}); // Prevent compaction ArgsManager bench_args; const auto chainParams = CreateChainParams(bench_args, CBaseChainParams::MAIN); @@ -49,5 +50,5 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) }); } -BENCHMARK(DeserializeBlockTest); -BENCHMARK(DeserializeAndCheckBlockTest); +BENCHMARK(DeserializeBlockTest, benchmark::PriorityLevel::HIGH); +BENCHMARK(DeserializeAndCheckBlockTest, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index d7b8c1badc66..8517c9fee29f 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -30,8 +30,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) struct PrevectorJob { prevector p; - PrevectorJob(){ - } + PrevectorJob() = default; explicit PrevectorJob(FastRandomContext& insecure_rand){ p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2)); } @@ -39,7 +38,10 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) { return true; } - void swap(PrevectorJob& x){p.swap(x.p);}; + void swap(PrevectorJob& x) noexcept + { + p.swap(x.p); + }; }; CCheckQueue queue {QUEUE_BATCH_SIZE}; // The main thread should be counted to prevent thread oversubscription, and @@ -68,4 +70,4 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) queue.StopWorkerThreads(); ECC_Stop(); } -BENCHMARK(CCheckQueueSpeedPrevectorJob); +BENCHMARK(CCheckQueueSpeedPrevectorJob, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 5beb833b48bf..53d89039a75b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2020 The Bitcoin Core developers +// Copyright (c) 2012-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,10 +6,24 @@ #include #include #include +#include #include #include +using node::NodeContext; +using wallet::AttemptSelection; +using wallet::CHANGE_LOWER; +using wallet::COutput; +using wallet::CWallet; +using wallet::CWalletTx; +using wallet::CoinEligibilityFilter; +using wallet::CoinSelectionParams; +using wallet::CreateDummyWalletDatabase; +using wallet::OutputGroup; +using wallet::SelectCoinsBnB; +using wallet::TxStateInactive; + static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector>& wtxs) { static int nextLockTime = 0; @@ -17,7 +31,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector(&wallet, MakeTransactionRef(std::move(tx)))); + wtxs.push_back(std::make_unique(MakeTransactionRef(std::move(tx)), TxStateInactive{})); } // Simple benchmark for wallet coin selection. Note that it maybe be necessary @@ -31,8 +45,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); - wallet.SetupLegacyScriptPubKeyMan(); + CWallet wallet(chain.get(), "", gArgs, CreateDummyWalletDatabase()); std::vector> wtxs; LOCK(wallet.cs_wallet); @@ -43,42 +56,42 @@ static void CoinSelection(benchmark::Bench& bench) addCoin(3 * COIN, wallet, wtxs); // Create coins - std::vector coins; + wallet::CoinsResult available_coins; for (const auto& wtx : wtxs) { - coins.emplace_back(wtx.get(), 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); + const auto txout = wtx->tx->vout.at(0); + available_coins.coins[OutputType::BECH32].emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); } const CoinEligibilityFilter filter_standard(1, 6, 0); - const CoinSelectionParams coin_selection_params(/* change_output_size= */ 34, - /* change_spend_size= */ 148, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_no_inputs_size= */ 0, /* avoid_partial= */ false); + FastRandomContext rand{}; + const CoinSelectionParams coin_selection_params{ + rand, + /*change_output_size=*/ 34, + /*change_spend_size=*/ 148, + /*min_change_target=*/ CHANGE_LOWER, + /*effective_feerate=*/ CFeeRate(0), + /*long_term_feerate=*/ CFeeRate(0), + /*discard_feerate=*/ CFeeRate(0), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; bench.run([&] { - std::set setCoinsRet; - CAmount nValueRet; - bool success = wallet.AttemptSelection(1003 * COIN, filter_standard, coins, setCoinsRet, nValueRet, coin_selection_params); - assert(success); - assert(nValueRet == 1003 * COIN); - assert(setCoinsRet.size() == 2); + auto result = AttemptSelection(wallet, 1003 * COIN, filter_standard, available_coins, coin_selection_params, /*allow_mixed_output_types=*/true); + assert(result); + assert(result->GetSelectedValue() == 1003 * COIN); + assert(result->GetInputSet().size() == 2); }); } -typedef std::set CoinSet; -static NodeContext testNode; -static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); -std::vector> wtxn; - // Copied from src/wallet/test/coinselector_tests.cpp static void add_coin(const CAmount& nValue, int nInput, std::vector& set) { CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - std::unique_ptr wtx = std::make_unique(&testWallet, MakeTransactionRef(std::move(tx))); + COutput output(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 0, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ true, /*fees=*/ 0); set.emplace_back(); - set.back().Insert(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); - wtxn.emplace_back(std::move(wtx)); + set.back().Insert(output, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } // Copied from src/wallet/test/coinselector_tests.cpp static CAmount make_hard_case(int utxos, std::vector& utxo_pool) @@ -96,21 +109,17 @@ static CAmount make_hard_case(int utxos, std::vector& utxo_pool) static void BnBExhaustion(benchmark::Bench& bench) { // Setup - testWallet.SetupLegacyScriptPubKeyMan(); std::vector utxo_pool; - CoinSet selection; - CAmount value_ret = 0; bench.run([&] { // Benchmark CAmount target = make_hard_case(17, utxo_pool); - SelectCoinsBnB(utxo_pool, target, 0, selection, value_ret); // Should exhaust + SelectCoinsBnB(utxo_pool, target, 0); // Should exhaust // Cleanup utxo_pool.clear(); - selection.clear(); }); } -BENCHMARK(CoinSelection); -BENCHMARK(BnBExhaustion); +BENCHMARK(CoinSelection, benchmark::PriorityLevel::HIGH); +BENCHMARK(BnBExhaustion, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 30fe11be6b30..162b02f344da 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -110,9 +110,9 @@ static void MuHash(benchmark::Bench& bench) { MuHash3072 acc; unsigned char key[32] = {0}; - int i = 0; + uint32_t i = 0; bench.run([&] { - key[0] = ++i; + key[0] = ++i & 0xFF; acc *= MuHash3072(key); }); } @@ -134,10 +134,6 @@ static void MuHashDiv(benchmark::Bench& bench) FastRandomContext rng(true); MuHash3072 muhash{rng.randbytes(32)}; - for (size_t i = 0; i < bench.epochIterations(); ++i) { - acc *= muhash; - } - bench.run([&] { acc /= muhash; }); @@ -154,19 +150,19 @@ static void MuHashPrecompute(benchmark::Bench& bench) }); } -BENCHMARK(RIPEMD160); -BENCHMARK(SHA1); -BENCHMARK(SHA256); -BENCHMARK(SHA512); -BENCHMARK(SHA3_256_1M); - -BENCHMARK(SHA256_32b); -BENCHMARK(SipHash_32b); -BENCHMARK(SHA256D64_1024); -BENCHMARK(FastRandom_32bit); -BENCHMARK(FastRandom_1bit); - -BENCHMARK(MuHash); -BENCHMARK(MuHashMul); -BENCHMARK(MuHashDiv); -BENCHMARK(MuHashPrecompute); +BENCHMARK(RIPEMD160, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA1, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA256, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA512, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA3_256_1M, benchmark::PriorityLevel::HIGH); + +BENCHMARK(SHA256_32b, benchmark::PriorityLevel::HIGH); +BENCHMARK(SipHash_32b, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA256D64_1024, benchmark::PriorityLevel::HIGH); +BENCHMARK(FastRandom_32bit, benchmark::PriorityLevel::HIGH); +BENCHMARK(FastRandom_1bit, benchmark::PriorityLevel::HIGH); + +BENCHMARK(MuHash, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashMul, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashDiv, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashPrecompute, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/data.cpp b/src/bench/data.cpp index 481e3721057e..35558b3aa78f 100644 --- a/src/bench/data.cpp +++ b/src/bench/data.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/descriptors.cpp b/src/bench/descriptors.cpp new file mode 100644 index 000000000000..972a6ff9531a --- /dev/null +++ b/src/bench/descriptors.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include