diff --git a/Makefile.am b/Makefile.am index ea2cba9fd2fa..051fca740ee1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -252,6 +252,7 @@ dist_noinst_SCRIPTS = autogen.sh EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) EXTRA_DIST += \ + test/bench \ test/functional \ test/fuzz @@ -319,5 +320,5 @@ clean-docs: clean-local: clean-docs rm -rf coverage_percent.txt test_dash.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP) - rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ + rm -rf test/bench/__pycache__ test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ rm -rf dist/ diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 9ca5ebe37216..0d8ac8312e53 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -34,11 +34,15 @@ if [ -z "${SKIP_PYTHON_INSTALL}" ]; then fi # NOTE: BUMP ALSO contrib/containers/ci/ci-slim.Dockerfile +${CI_RETRY_EXE} pip3 install aiohttp==3.13.3 ${CI_RETRY_EXE} pip3 install codespell==2.2.1 ${CI_RETRY_EXE} pip3 install flake8==5.0.4 +${CI_RETRY_EXE} pip3 install jinja2==3.1.6 ${CI_RETRY_EXE} pip3 install lief==0.13.2 +${CI_RETRY_EXE} pip3 install multiprocess==0.70.19 ${CI_RETRY_EXE} pip3 install mypy==0.981 ${CI_RETRY_EXE} pip3 install pyzmq==24.0.1 +${CI_RETRY_EXE} pip3 install tabulate==0.10.0 ${CI_RETRY_EXE} pip3 install vulture==2.6 SHELLCHECK_VERSION=v0.8.0 diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 3e083c3d4d62..8c9873305c26 100755 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -11,4 +11,4 @@ export HOST=x86_64-pc-linux-gnu export PACKAGES="python3-zmq" export DEP_OPTS="NO_WALLET=1 CC=gcc-14 CXX=g++-14" export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports CC=gcc-14 CXX=g++-14" +export BITCOIN_CONFIG="--enable-reduce-exports --with-drogon=no CC=gcc-14 CXX=g++-14" diff --git a/configure.ac b/configure.ac index 31951e1f7c47..c69ae38b1287 100644 --- a/configure.ac +++ b/configure.ac @@ -262,6 +262,12 @@ AC_ARG_ENABLE([multiprocess], [enable_multiprocess=$enableval], [enable_multiprocess=no]) +AC_ARG_WITH([drogon], + [AS_HELP_STRING([--with-drogon=yes|no|auto], + [enable drogon server support (default: auto, i.e., enabled if drogon is found)])], + [use_drogon=$withval], + [use_drogon=auto]) + AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [do not install man pages (default is to install)])],, @@ -1455,6 +1461,7 @@ if test "$enable_fuzz" = "yes"; then use_upnp=no use_natpmp=no use_zmq=no + use_drogon=no enable_fuzz_binary=yes AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) @@ -1547,6 +1554,7 @@ if test "$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nononono"; th use_upnp=no use_natpmp=no use_zmq=no + use_drogon=no fi dnl Check for libminiupnpc (optional) @@ -1721,6 +1729,35 @@ LDFLAGS="$TEMP_LDFLAGS" AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) +dnl Drogon server check +if test "$use_drogon" != "no"; then + AC_CHECK_HEADER([drogon/drogon.h], [have_drogon=yes], [have_drogon=no]) +fi +AC_MSG_CHECKING([whether to build with drogon server support]) +if test "$use_drogon" = "no"; then + use_drogon=no +elif test "$have_drogon" = "no"; then + if test "$use_drogon" = "yes"; then + AC_MSG_ERROR([drogon support requested but headers not found. Use --without-drogon]) + fi + use_drogon=no +else + AC_DEFINE([USE_DROGON], [1], [Define this symbol to enable the Drogon server]) + DROGON_LIBS="-ldrogon -ltrantor -ljsoncpp -lz" + case $host in + *linux*) + DROGON_LIBS="$DROGON_LIBS -ldl -luuid" + ;; + *darwin*) + DROGON_LIBS="$DROGON_LIBS -ldl" + ;; + esac + use_drogon=yes +fi +AC_MSG_RESULT([$use_drogon]) + +AM_CONDITIONAL([USE_DROGON], [test "$use_drogon" = "yes"]) + dnl libmultiprocess library check libmultiprocess_found=no @@ -2043,6 +2080,7 @@ AC_SUBST(BACKTRACE_FLAGS) AC_SUBST(BACKTRACE_LDFLAGS) AC_SUBST(BACKTRACE_LIBS) AC_SUBST(GMP_LIBS) +AC_SUBST(DROGON_LIBS) AC_SUBST(NATPMP_CPPFLAGS) AC_SUBST(NATPMP_LIBS) AC_SUBST(DSYMUTIL_FLAT) @@ -2062,6 +2100,7 @@ AC_CONFIG_LINKS([src/.bear-tidy-config:src/.bear-tidy-config]) AC_CONFIG_LINKS([src/.clang-tidy:src/.clang-tidy]) AC_CONFIG_LINKS([src/ipc/.clang-tidy:src/ipc/.clang-tidy]) AC_CONFIG_LINKS([src/test/.clang-tidy:src/test/.clang-tidy]) +AC_CONFIG_LINKS([test/bench/bench_runner.py:test/bench/bench_runner.py]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py]) @@ -2106,6 +2145,7 @@ echo "Options used to compile and link:" echo " external signer = $use_external_signer" echo " multiprocess = $build_multiprocess" echo " with libs = $build_bitcoin_libs" +echo " with drogon = $use_drogon" echo " with wallet = $enable_wallet" if test "$enable_wallet" != "no"; then echo " with sqlite = $use_sqlite" diff --git a/contrib/containers/ci/ci-slim.Dockerfile b/contrib/containers/ci/ci-slim.Dockerfile index 5332f8504a57..a2f13d28ea69 100644 --- a/contrib/containers/ci/ci-slim.Dockerfile +++ b/contrib/containers/ci/ci-slim.Dockerfile @@ -77,13 +77,15 @@ ENV UV_SYSTEM_PYTHON=1 # Install Python packages # NOTE: if versions are changed, update ci/lint/04_install.sh RUN uv pip install --system --break-system-packages \ + aiohttp==3.13.3 \ codespell==2.2.1 \ flake8==5.0.4 \ - jinja2 \ + jinja2==3.1.6 \ lief==0.13.2 \ - multiprocess \ + multiprocess==0.70.19 \ mypy==0.981 \ pyzmq==24.0.1 \ + tabulate==0.10.0 \ vulture==2.6 # Install packages relied on by tests diff --git a/depends/packages/drogon.mk b/depends/packages/drogon.mk new file mode 100644 index 000000000000..13b00d676454 --- /dev/null +++ b/depends/packages/drogon.mk @@ -0,0 +1,39 @@ +package=drogon +$(package)_version=1.9.12 +$(package)_download_path=https://github.com/drogonframework/drogon/archive/refs/tags +$(package)_download_file=v$($(package)_version).tar.gz +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=becc3c4f3b90f069f814baef164a7e3a2b31476dc6fe249b02ff07a13d032f48 +$(package)_dependencies=jsoncpp trantor zlib +$(package)_build_subdir=build + +define $(package)_set_vars + $(package)_config_opts := -DCMAKE_BUILD_TYPE=None + $(package)_config_opts += -DBUILD_BROTLI=OFF + $(package)_config_opts += -DBUILD_CTL=OFF + $(package)_config_opts += -DBUILD_EXAMPLES=OFF + $(package)_config_opts += -DBUILD_ORM=OFF + $(package)_config_opts += -DBUILD_REDIS=OFF + $(package)_config_opts += -DBUILD_SHARED_LIBS=OFF + $(package)_config_opts += -DBUILD_TESTING=OFF + $(package)_config_opts += -DBUILD_YAML_CONFIG=OFF + $(package)_config_opts += -DUSE_SUBMODULE=OFF + $(package)_cxxflags += -fdebug-prefix-map=$($(package)_extract_dir)=/usr -fmacro-prefix-map=$($(package)_extract_dir)=/usr +endef + +define $(package)_config_cmds + $($(package)_cmake) -S .. -B . +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf lib/cmake && \ + rm -rf lib/pkgconfig +endef diff --git a/depends/packages/jsoncpp.mk b/depends/packages/jsoncpp.mk new file mode 100644 index 000000000000..3dc0c43db087 --- /dev/null +++ b/depends/packages/jsoncpp.mk @@ -0,0 +1,32 @@ +package=jsoncpp +$(package)_version=1.9.6 +$(package)_download_path=https://github.com/open-source-parsers/jsoncpp/archive/refs/tags +$(package)_download_file=$($(package)_version).tar.gz +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=f93b6dd7ce796b13d02c108bc9f79812245a82e577581c4c9aabe57075c90ea2 +$(package)_build_subdir=build + +define $(package)_set_vars + $(package)_config_opts := -DCMAKE_BUILD_TYPE=None + $(package)_config_opts += -DBUILD_SHARED_LIBS=OFF + $(package)_config_opts += -DBUILD_STATIC_LIBS=ON + $(package)_config_opts += -DJSONCPP_WITH_CMAKE_PACKAGE=ON + $(package)_config_opts += -DJSONCPP_WITH_POST_BUILD_UNITTEST=OFF + $(package)_config_opts += -DJSONCPP_WITH_TESTS=OFF +endef + +define $(package)_config_cmds + $($(package)_cmake) -S .. -B . +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf lib/pkgconfig +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 7e0bb2633219..aa68e44ca640 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,4 +1,8 @@ -packages:=gmp backtrace +backtrace_packages:=backtrace +drogon_packages:=drogon jsoncpp trantor zlib +gmp_packages:=gmp + +packages:=$(backtrace_packages) $(drogon_packages) $(gmp_packages) boost_packages = boost diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 18aedcf7afcf..51a357bbf1af 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -4,6 +4,7 @@ $(package)_download_path=https://download.qt.io/archive/qt/5.15/$($(package)_ver $(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) $(package)_sha256_hash=7b632550ea1048fc10c741e46e2e3b093e5ca94dfa6209e9e0848800e247023b +$(package)_dependencies=zlib $(package)_linux_dependencies := freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_freebsd_dependencies := $($(package)_linux_dependencies) $(package)_qt_libs=corelib network widgets gui plugins testlib @@ -91,7 +92,7 @@ $(package)_config_opts += -prefix $(host_prefix) $(package)_config_opts += -qt-libpng $(package)_config_opts += -qt-pcre $(package)_config_opts += -qt-harfbuzz -$(package)_config_opts += -qt-zlib +$(package)_config_opts += -system-zlib $(package)_config_opts += -static $(package)_config_opts += -v $(package)_config_opts += -no-feature-bearermanagement @@ -279,7 +280,9 @@ endef define $(package)_config_cmds cd qtbase && \ - ./configure -top-level $($(package)_config_opts) + ./configure -top-level $($(package)_config_opts) && \ + echo "host_build: QT_CONFIG ~= s/system-zlib/zlib" >> mkspecs/qconfig.pri && \ + echo "CONFIG += force_bootstrap" >> mkspecs/qconfig.pri endef define $(package)_build_cmds diff --git a/depends/packages/trantor.mk b/depends/packages/trantor.mk new file mode 100644 index 000000000000..aec13c7d1af8 --- /dev/null +++ b/depends/packages/trantor.mk @@ -0,0 +1,32 @@ +package=trantor +$(package)_version=1.5.26 +$(package)_download_path=https://github.com/an-tao/trantor/archive/refs/tags +$(package)_download_file=v$($(package)_version).tar.gz +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=e47092938aaf53d51c8bc72d8f54ebdcf537e6e4ac9c8276f3539413d6dfeddf +$(package)_build_subdir=build + +define $(package)_set_vars + $(package)_config_opts := -DCMAKE_BUILD_TYPE=None + $(package)_config_opts += -DBUILD_C-ARES=OFF + $(package)_config_opts += -DBUILD_SHARED_LIBS=OFF + $(package)_config_opts += -DBUILD_TESTING=OFF + $(package)_config_opts += -DTRANTOR_USE_TLS=none + $(package)_cxxflags += -fdebug-prefix-map=$($(package)_extract_dir)=/usr -fmacro-prefix-map=$($(package)_extract_dir)=/usr +endef + +define $(package)_config_cmds + $($(package)_cmake) -S .. -B . +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf lib/pkgconfig +endef diff --git a/depends/packages/zlib.mk b/depends/packages/zlib.mk new file mode 100644 index 000000000000..1158ce65ed96 --- /dev/null +++ b/depends/packages/zlib.mk @@ -0,0 +1,30 @@ +package=zlib +$(package)_version=1.3.2 +$(package)_download_path=https://www.zlib.net +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16 + +define $(package)_set_vars + $(package)_config_opts := CC="$($(package)_cc)" + $(package)_config_opts += CFLAGS="$($(package)_cflags) $($(package)_cppflags) -fPIC" + $(package)_config_opts += RANLIB="$($(package)_ranlib)" + $(package)_config_opts += AR="$($(package)_ar)" + $(package)_config_opts_darwin += AR="$($(package)_libtool)" + $(package)_config_opts_darwin += ARFLAGS="-o" + $(package)_config_opts_android += CHOST=$(host) +endef + +# zlib has its own custom configure script that takes in options like CC, +# CFLAGS, RANLIB, AR, and ARFLAGS from the environment rather than from +# command-line arguments. +define $(package)_config_cmds + env $($(package)_config_opts) ./configure --static --prefix=$(host_prefix) +endef + +define $(package)_build_cmds + $(MAKE) libz.a +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 4b95ec1e3210..1046b53c120c 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -3,8 +3,20 @@ Unauthenticated REST Interface The REST API can be enabled with the `-rest` option. -The interface runs on the same port as the JSON-RPC interface, by default port 9998 for mainnet, port 19998 for testnet, -port 19788 for devnet, and port 19898 for regtest. +The interface runs on its own dedicated port, separate from the JSON-RPC interface. Default port 9997 for mainnet, port 19997 for testnet, port 19897 for regtest and port 19797 for devnet. The port can be overridden with `-restport=`. + +Configuration +------------- + +| Option | Default | Description | +|---------------------------|-------------|--------------------------------------------------------------| +| `-rest` | `0` | Enable the REST server. | +| `-restbind=` | `127.0.0.1` | IP address to bind to. | +| `-restport=` | per-network | Port to listen on (1–65535). | +| `-restthreads=` | `4` | Number of I/O threads. | +| `-restmaxconnections=` | `100` | Maximum concurrent connections (1–65535). | +| `-restidletimeout=` | `30` | Seconds before an idle connection is closed (5–3600). | +| `-restreuseport` | `false` | Allow multiple sockets to bind to the same port (Linux only).| REST Interface consistency guarantees ------------------------------------- @@ -12,18 +24,6 @@ REST Interface consistency guarantees The [same guarantees as for the RPC Interface](/doc/JSON-RPC-interface.md#rpc-consistency-guarantees) apply. -Limitations ------------ - -There is a known issue in the REST interface that can cause a node to crash if -too many http connections are being opened at the same time because the system runs -out of available file descriptors. To prevent this from happening you might -want to increase the number of maximum allowed file descriptors in your system -and try to prevent opening too many connections to your rest interface at the -same time if this is under your control. It is hard to give general advice -since this depends on your system but if you make several hundred requests at -once you are definitely at risk of encountering this issue. - Supported API ------------- @@ -97,7 +97,7 @@ input and output serialization (relevant for `bin` and `hex` output formats). Example: ``` -$ curl localhost:19998/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff7627ff72e5e8b0f71210f92ea7a4000c5d75-0.json 2>/dev/null | json_pp +$ curl localhost:19997/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff7627ff72e5e8b0f71210f92ea7a4000c5d75-0.json 2>/dev/null | json_pp { "chainHeight" : 325347, "chaintipHash" : "00000000fb01a7f3745a717f8caebee056c484e6e0bfe4a9591c235bb70506fb", @@ -133,4 +133,4 @@ Refer to the `getrawmempool` RPC help for details. Risks ------------- -Running a web browser on the same node with a REST enabled dashd can be a risk. Accessing prepared XSS websites could read out tx/block data of your node by placing links like `