diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..3d843db6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +release +install +build +result +**/.nix-install +**/.nix-pip +**/__pycache__ +**/*.egg-info +**/*.so +**/*.dylib +**/dist diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..179e0ce1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,148 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + - master + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + nix-build: + name: Nix BTC checks + runs-on: ubuntu-24.04 + timeout-minutes: 120 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v31 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + extra_nix_config: | + experimental-features = nix-command flakes + accept-flake-config = true + + - name: Run flake checks + run: nix flake check -L --cores 1 --max-jobs 1 + + native-linux: + name: Native Linux build + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + PREFIX: ${{ github.workspace }}/install + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential ca-certificates cmake git ninja-build pkg-config \ + libboost-all-dev libssl-dev librocksdb-dev libsparsehash-dev \ + libsecp256k1-dev librange-v3-dev nlohmann-json3-dev libcereal-dev \ + libgtest-dev libcurl4-openssl-dev libjsoncpp-dev \ + python3 python3-dev python3-pip python3-venv + + - name: Build unpackaged dependencies + run: ./scripts/install-unpackaged-deps.sh "$PREFIX" + + - name: Build and install C++ targets + run: | + cmake -S . -B release -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX" + cmake --build release --parallel 2 + cmake --install release + + - name: Build Python bindings + run: | + python3 -m venv "$PREFIX/venv" + "$PREFIX/venv/bin/python" -m pip install --upgrade pip setuptools wheel + CMAKE_PREFIX_PATH="$PREFIX" \ + CMAKE_BUILD_PARALLEL_LEVEL=2 \ + "$PREFIX/venv/bin/python" -m pip install ./blockscipy + + - name: Smoke test native Linux build + run: | + export PATH="$PREFIX/bin:$PREFIX/venv/bin:$PATH" + export LD_LIBRARY_PATH="$PREFIX/lib:${LD_LIBRARY_PATH:-}" + blocksci_parser --help >/dev/null + python -c 'import blocksci; print(blocksci.VERSION)' + + native-macos: + name: Native macOS arm64 build + runs-on: macos-15 + timeout-minutes: 120 + env: + PREFIX: ${{ github.workspace }}/install + BLOCKSCI_BUILD_ROCKSDB: "1" + HOMEBREW_NO_AUTO_UPDATE: "1" + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "1" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + brew install \ + cmake ninja git boost openssl google-sparsehash secp256k1 \ + range-v3 nlohmann-json cereal googletest curl jsoncpp \ + bzip2 lz4 snappy zstd + + - name: Build unpackaged and pinned dependencies + run: ./scripts/install-unpackaged-deps.sh "$PREFIX" + + - name: Build and install C++ targets + run: | + brew_prefix="$(brew --prefix)" + openssl_prefix="$(brew --prefix openssl)" + cmake -S . -B release -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX;$brew_prefix" \ + -DOPENSSL_ROOT_DIR="$openssl_prefix" + cmake --build release --parallel 2 + cmake --install release + + - name: Build Python bindings + run: | + brew_prefix="$(brew --prefix)" + openssl_prefix="$(brew --prefix openssl)" + python3 -m venv "$PREFIX/venv" + "$PREFIX/venv/bin/python" -m pip install --upgrade \ + pip setuptools wheel scikit-build-core pybind11 + pybind11_dir="$("$PREFIX/venv/bin/python" -c 'import pybind11; print(pybind11.get_cmake_dir())')" + test -f "$PREFIX/lib/cmake/blocksci/blocksci-config.cmake" + test -f "$pybind11_dir/pybind11Config.cmake" + CMAKE_BUILD_PARALLEL_LEVEL=2 \ + "$PREFIX/venv/bin/python" -m pip install -v --no-build-isolation ./blockscipy \ + --config-settings=cmake.define.CMAKE_PREFIX_PATH="$PREFIX;$brew_prefix;$pybind11_dir" \ + --config-settings=cmake.define.blocksci_DIR="$PREFIX/lib/cmake/blocksci" \ + --config-settings=cmake.define.pybind11_DIR="$pybind11_dir" \ + --config-settings=cmake.define.OPENSSL_ROOT_DIR="$openssl_prefix" \ + --config-settings=build.verbose=true + + - name: Smoke test native macOS build + run: | + export PATH="$PREFIX/bin:$PREFIX/venv/bin:$PATH" + export DYLD_LIBRARY_PATH="$PREFIX/lib:${DYLD_LIBRARY_PATH:-}" + blocksci_parser --help >/dev/null + python -c 'import blocksci; print(blocksci.VERSION)' diff --git a/.gitignore b/.gitignore index 0a28194a..abd0cad7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,20 @@ release +install xcode debug build +bin/ +lib/ .mtime_cache *.DS_Store +*.log Notebooks/blocksci *.so *.dylib *.pyc +__pycache__/ include/blocksci/external +include/blocksci/blocksci_export.h python/blocksci/pybind11 *.egg-info blockscipy/dist/* @@ -16,5 +22,10 @@ blockscipy/dist/* .idea .benchmarks result +result-* .nix-install .nix-pip +LOCAL_PIP/ +test-chain/ +/.direnv +/.envrc diff --git a/.gitmodules b/.gitmodules index bebab775..40889ddd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "Notebooks/blocksci/Blockchain-Known-Pools"] - path = blockscipy/blocksci/Blockchain-Known-Pools - url = https://github.com/blockchain/Blockchain-Known-Pools [submodule "test/testchain-generator"] path = test/testchain-generator url = https://github.com/citp/testchain-generator diff --git a/blockscipy/CMakeLists.txt b/blockscipy/CMakeLists.txt index 52363875..454cb8d7 100644 --- a/blockscipy/CMakeLists.txt +++ b/blockscipy/CMakeLists.txt @@ -30,7 +30,7 @@ file(GLOB PYTHON_INTERFACE_GENERIC_PROXY_SOURCES "src/generic_proxy/*.cpp") file(GLOB PYTHON_INTERFACE_HEADERS "src/*.hpp") file(GLOB PYTHON_INTERFACE_SOURCES "src/*.cpp") -pybind11_add_module(_blocksci ${PYTHON_INTERFACE_PROXY_HEADERS} ${PYTHON_INTERFACE_PROXY_SOURCES} ${PYTHON_CHAIN_SOURCES} ${PYTHON_CLUSTER_HEADERS} ${PYTHON_CLUSTER_SOURCES} ${PYTHON_CHAIN_HEADERS} ${PYTHON_SCRIPTS_HEADERS} ${PYTHON_SCRIPTS_SOURCES} ${PYTHON_INTERFACE_SOURCES} ${PYTHON_INTERFACE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_SOURCES} ${PYTHON_INTERFACE_GENERIC_PROXY_HEADERS} ${PYTHON_INTERFACE_GENERIC_PROXY_SOURCES}) +pybind11_add_module(_blocksci NO_EXTRAS ${PYTHON_INTERFACE_PROXY_HEADERS} ${PYTHON_INTERFACE_PROXY_SOURCES} ${PYTHON_CHAIN_SOURCES} ${PYTHON_CLUSTER_HEADERS} ${PYTHON_CLUSTER_SOURCES} ${PYTHON_CHAIN_HEADERS} ${PYTHON_SCRIPTS_HEADERS} ${PYTHON_SCRIPTS_SOURCES} ${PYTHON_INTERFACE_SOURCES} ${PYTHON_INTERFACE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_SOURCES} ${PYTHON_INTERFACE_GENERIC_PROXY_HEADERS} ${PYTHON_INTERFACE_GENERIC_PROXY_SOURCES}) set_target_properties(_blocksci PROPERTIES CXX_VISIBILITY_PRESET hidden) set_target_properties(_blocksci PROPERTIES VISIBILITY_INLINES_HIDDEN 1) @@ -39,6 +39,10 @@ add_subdirectory(external) target_compile_options(_blocksci PRIVATE -Wall -Wextra -Wpedantic) +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_compile_options(_blocksci PRIVATE -O1) +endif() + target_link_libraries( _blocksci diff --git a/blockscipy/blocksci/Blockchain-Known-Pools b/blockscipy/blocksci/Blockchain-Known-Pools deleted file mode 160000 index 29ab27c8..00000000 --- a/blockscipy/blocksci/Blockchain-Known-Pools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 29ab27c844ebdb63110f8783f73b9decd4abc221 diff --git a/blockscipy/blocksci/__init__.py b/blockscipy/blocksci/__init__.py index 809786f2..485b1225 100644 --- a/blockscipy/blocksci/__init__.py +++ b/blockscipy/blocksci/__init__.py @@ -19,10 +19,10 @@ from functools import reduce import dateparser +import multiprocess as mp import pandas as pd import psutil from dateutil.relativedelta import relativedelta -from multiprocess import Pool from ._blocksci import * from ._blocksci import _traverse @@ -39,6 +39,48 @@ sys.modules["blocksci.heuristics"] = heuristics sys.modules["blocksci.heuristics.change"] = heuristics.change +# pybind11 3 gives enums Python-native ``Enum`` formatting by default +# (``address_type.pubkeyhash`` / ````). +# BlockSci's public API and regression fixtures historically exposed a +# human-readable ``str()`` and a compact qualified-name ``repr()``. +_ADDRESS_TYPE_LABELS = { + address_type.pubkey: "Pay to pubkey", + address_type.pubkeyhash: "Pay to pubkey hash", + address_type.scripthash: "Pay to script hash", + address_type.multisig: "Multisig", + address_type.multisig_pubkey: "Multisig Public Key", + address_type.nonstandard: "Nonstandard", + address_type.nulldata: "Null data", + address_type.witness_pubkeyhash: "Pay to witness pubkey hash", + address_type.witness_scripthash: "Pay to witness script hash", + address_type.witness_unknown: "Pay to witness unknown", +} + +_ADDRESS_TYPE_REPR_NAMES = { + address_type.pubkey: "pubkey", + address_type.pubkeyhash: "pubkeyhash", + address_type.scripthash: "scripthash", + address_type.multisig: "multisig", + address_type.multisig_pubkey: "multisig_pubkey", + address_type.nonstandard: "nonstandard", + address_type.nulldata: "nulldata", + address_type.witness_pubkeyhash: "witness_pubkeyhash", + address_type.witness_scripthash: "witness_scripthash", + address_type.witness_unknown: "witness_unknown", +} + + +def _address_type_str(value): + return _ADDRESS_TYPE_LABELS.get(value, "Unknown Address Type") + + +def _address_type_repr(value): + return f"address_type.{_ADDRESS_TYPE_REPR_NAMES.get(value, 'unknown')}" + + +address_type.__str__ = _address_type_str +address_type.__repr__ = _address_type_repr + class _NoDefault: def __repr__(self): @@ -48,6 +90,15 @@ def __repr__(self): MISSING_PARAM = _NoDefault() +def _multiprocess_context(): + # Python 3.12+ warns when forking a multithreaded process. Pytest and + # native extension imports can start helper threads, so prefer spawn on + # Darwin where fork-without-exec is particularly fragile. + if sys.platform == "darwin" and sys.version_info >= (3, 12): + return mp.get_context("spawn") + return mp.get_context() + + # Alert the user if the disk space is getting full disk_info = os.statvfs("/") free_space = (disk_info.f_frsize * disk_info.f_bavail) // (1024**3) @@ -96,7 +147,7 @@ def real_map_func(input): file.seek(0) return file - with Pool(cpu_count - 1) as p: + with _multiprocess_context().Pool(cpu_count - 1) as p: results_future = p.map_async(real_map_func, segments[1:]) first = map_func(chain[raw_segments[0][0] : raw_segments[0][1]]) results = results_future.get() @@ -865,6 +916,21 @@ class DummyClass: loaderDirectory = os.path.dirname(os.path.abspath(inspect.getsourcefile(DummyClass))) +def _load_pool_data(): + import json + + pools_path = os.environ.get( + "BLOCKSCI_POOLS_JSON", + os.path.join(loaderDirectory, "Blockchain-Known-Pools", "pools.json"), + ) + try: + with open(pools_path) as f: + return json.load(f) + except FileNotFoundError: + logging.warning("BlockSci miner labels disabled: pools.json not found at %s", pools_path) + return {"payout_addresses": {}, "coinbase_tags": {}} + + def get_miner(block) -> str: """ Get the miner of the block based on the text in the coinbase transaction @@ -874,20 +940,18 @@ def get_miner(block) -> str: global pool_data global coinbase_tag_re if first_miner_run: - import json - - with open(loaderDirectory + "/Blockchain-Known-Pools/pools.json") as f: - pool_data = json.load(f) - addresses = [block._access.address_from_string(addr_string) for addr_string in pool_data["payout_addresses"]] - tagged_addresses = { - pointer: pool_data["payout_addresses"][address] - for address in addresses - if address in pool_data["payout_addresses"] - } - coinbase_tag_re = re.compile("|".join(map(re.escape, pool_data["coinbase_tags"]))) + pool_data = _load_pool_data() + tagged_addresses = {} + for addr_string, pool_info in pool_data.get("payout_addresses", {}).items(): + try: + tagged_addresses[block._access.address_from_string(addr_string)] = pool_info + except Exception: + logging.debug("Skipping invalid miner payout address %s", addr_string, exc_info=True) + coinbase_tags = pool_data.get("coinbase_tags", {}) + coinbase_tag_re = re.compile("|".join(map(re.escape, coinbase_tags))) if coinbase_tags else None first_miner_run = False coinbase = block.coinbase_param.decode("utf_8", "replace") - tag_matches = re.findall(coinbase_tag_re, coinbase) + tag_matches = re.findall(coinbase_tag_re, coinbase) if coinbase_tag_re is not None else [] if tag_matches: return pool_data["coinbase_tags"][tag_matches[0]]["name"] for txout in block.coinbase_tx.outs: diff --git a/blockscipy/src/blocksci_range_type.cpp b/blockscipy/src/blocksci_range_type.cpp index 3847a405..74c95d5a 100644 --- a/blockscipy/src/blocksci_range_type.cpp +++ b/blockscipy/src/blocksci_range_type.cpp @@ -29,7 +29,7 @@ struct BlocksciRangeTypeObjectCaster { struct BlockSciRangeToIterator { template BlocksciIteratorType operator()(T &&t) { - return BlocksciIteratorType{RawIterator>{std::forward(t)}}; + return BlocksciIteratorType{RawIterator>{std::forward(t)}}; } }; diff --git a/blockscipy/src/blocksci_type_converter.cpp b/blockscipy/src/blocksci_type_converter.cpp index 700ba159..e6939bc3 100644 --- a/blockscipy/src/blocksci_type_converter.cpp +++ b/blockscipy/src/blocksci_type_converter.cpp @@ -20,18 +20,18 @@ #include -blocksci::AnyScript BlockSciTypeConverter::operator()(const blocksci::Address &address) { +blocksci::AnyScript BlockSciTypeConverter::operator()(const blocksci::Address &address) const { return address.getScript(); } -RawRange BlockSciTypeConverter::operator()(const blocksci::InputRange &val) { +RawRange BlockSciTypeConverter::operator()(const blocksci::InputRange &val) const { return ranges::any_view{val}; } -RawRange BlockSciTypeConverter::operator()(const blocksci::OutputRange &val) { +RawRange BlockSciTypeConverter::operator()(const blocksci::OutputRange &val) const { return ranges::any_view{val}; } -RawRange BlockSciTypeConverter::operator()(const blocksci::BlockRange &val) { +RawRange BlockSciTypeConverter::operator()(const blocksci::BlockRange &val) const { return ranges::any_view{val}; -} \ No newline at end of file +} diff --git a/blockscipy/src/blocksci_type_converter.hpp b/blockscipy/src/blocksci_type_converter.hpp index bbecf838..3d122ff9 100644 --- a/blockscipy/src/blocksci_type_converter.hpp +++ b/blockscipy/src/blocksci_type_converter.hpp @@ -22,39 +22,39 @@ struct BlockSciTypeConverter { - template T operator()(const T &t) { + template T operator()(const T &t) const { return t; } - blocksci::AnyScript operator()(const blocksci::Address &address); + blocksci::AnyScript operator()(const blocksci::Address &address) const; - int64_t operator()(uint16_t val) { + int64_t operator()(uint16_t val) const { return static_cast(val); } - int64_t operator()(int16_t val) { + int64_t operator()(int16_t val) const { return static_cast(val); } - int64_t operator()(uint32_t val) { + int64_t operator()(uint32_t val) const { return static_cast(val); } - int64_t operator()(int32_t val) { + int64_t operator()(int32_t val) const { return static_cast(val); } // Potential overflow - int64_t operator()(uint64_t val) { + int64_t operator()(uint64_t val) const { return static_cast(val); } - RawRange operator()(const blocksci::InputRange &val); - RawRange operator()(const blocksci::OutputRange &val); - RawRange operator()(const blocksci::BlockRange &val); + RawRange operator()(const blocksci::InputRange &val) const; + RawRange operator()(const blocksci::OutputRange &val) const; + RawRange operator()(const blocksci::BlockRange &val) const; template - auto operator()(const ranges::optional &val) -> ranges::optionaloperator()(*val))> { + auto operator()(const ranges::optional &val) const -> ranges::optionaloperator()(*val))> { if (val) { return this->operator()(*val); } else { @@ -62,11 +62,11 @@ struct BlockSciTypeConverter { } } - template auto operator()(Iterator &val) -> Iteratoroperator()(*val))> { + template auto operator()(Iterator &val) const -> Iteratoroperator()(*val))> { return val | ranges::views::transform(this); } - template auto operator()(Range &val) -> Rangeoperator()(*val))> { + template auto operator()(Range &val) const -> Rangeoperator()(*val))> { return val | ranges::views::transform(this); } }; diff --git a/blockscipy/src/proxy/range.hpp b/blockscipy/src/proxy/range.hpp index 162d3b73..4d75ef8e 100644 --- a/blockscipy/src/proxy/range.hpp +++ b/blockscipy/src/proxy/range.hpp @@ -97,8 +97,8 @@ template void setupRangesProxy(AllProxyClasses if (!slice.compute(chainSize, &start, &stop, &step, &slicelength)) throw pybind11::error_already_set(); - auto subset = range[{static_cast>>(start), - static_cast>>(stop)}]; + auto subset = range[{static_cast>>(start), + static_cast>>(stop)}]; return subset | ranges::views::stride(step); }); }, diff --git a/blockscipy/src/python_range_conversion.cpp b/blockscipy/src/python_range_conversion.cpp index 2753c976..19ea78d0 100644 --- a/blockscipy/src/python_range_conversion.cpp +++ b/blockscipy/src/python_range_conversion.cpp @@ -37,48 +37,49 @@ namespace py = pybind11; struct NumpyConverter { - template T operator()(const T &val) { + template T operator()(const T &val) const { return val; } - NumpyDatetime operator()(const std::chrono::system_clock::time_point &val) { + NumpyDatetime operator()(const std::chrono::system_clock::time_point &val) const { return {std::chrono::duration_cast(val.time_since_epoch()).count()}; } - std::array operator()(const blocksci::uint256 &val) { + std::array operator()(const blocksci::uint256 &val) const { auto hexStr = val.GetHex(); std::array ret; std::copy_n(hexStr.begin(), 64, ret.begin()); return ret; } - std::array operator()(const blocksci::uint160 &val) { + std::array operator()(const blocksci::uint160 &val) const { auto hexStr = val.GetHex(); std::array ret; std::copy_n(hexStr.begin(), 40, ret.begin()); return ret; } - NumpyBool operator()(const bool &val) { + NumpyBool operator()(const bool &val) const { return {val}; } }; template -pybind11::array_t>()))> +pybind11::array_t>()))> convertRandomSizedNumpy(T &&t) { auto numpy_converted = ranges::views::transform(std::move(t), NumpyConverter{}); auto rangeSize = static_cast(ranges::size(numpy_converted)); - pybind11::array_t> ret{rangeSize}; + pybind11::array_t> ret{ + static_cast(rangeSize)}; auto retPtr = ret.mutable_data(); ranges::copy(numpy_converted, retPtr); return ret; } template -pybind11::array_t>()))> convertInputNumpy(T &&t) { +pybind11::array_t>()))> convertInputNumpy(T &&t) { auto ret = ranges::to(ranges::views::transform(std::move(t), NumpyConverter{})); - return pybind11::array_t{ret.size(), ret.data()}; + return pybind11::array_t{static_cast(ret.size()), ret.data()}; } template py::list convertRandomSizedPy(T &&t) { @@ -101,15 +102,15 @@ template py::list convertInputPy(T &&t) { } template -Iterator>()))> convertInputBlockSci(T &&t) { - return ranges::any_view>()))>{ +Iterator>()))> convertInputBlockSci(T &&t) { + return ranges::any_view>()))>{ ranges::views::transform(std::forward(t), BlockSciTypeConverter{})}; } template -Range>()))> +Range>()))> convertRandomSizedBlockSci(T &&t) { - return ranges::any_view>())), + return ranges::any_view>())), random_access_sized>{ranges::views::transform(std::forward(t), BlockSciTypeConverter{})}; } diff --git a/build.md b/build.md new file mode 100644 index 00000000..e2128a77 --- /dev/null +++ b/build.md @@ -0,0 +1,167 @@ +# Building BlockSci from source + +This guide builds BlockSci on Linux or macOS using your system toolchain and +package manager, for environments where [Nix](nix.md) is not available. + +The Nix flake remains the canonical, reproducible build — it pins every +dependency and is what CI builds. Prefer it when you can; see [nix.md](nix.md). +This document is the supported manual path for everyone else. + +BlockSci is built in two halves: + +- `libblocksci` and the command-line tools (`blocksci_parser`, + `blocksci_clusterer`, …), built with CMake. +- `blockscipy`, the Python bindings, built on top of the installed C++ library + via `scikit-build-core`. + +## Requirements + +- A C++17 compiler (recent GCC or Clang) +- CMake ≥ 3.15 +- Python ≥ 3.11 with `pip` +- `git` + +Both halves and the from-source dependencies install into a single prefix so +they find each other. By default we use a disposable, project-local prefix — run +this from the repository root: + +```bash +export PREFIX="$PWD/install" +``` + +For a system-wide install, use `PREFIX=/usr/local` with `sudo`; it's on the +default loader path, so you can skip the `LD_LIBRARY_PATH` step at the end. + +## Dependencies + +BlockSci's dependencies fall into two groups: those your package manager +provides, and a handful that are not packaged and are built from source by a +helper script. + +### System packages + +#### **Debian / Ubuntu:** + +```bash +sudo apt install \ + build-essential cmake git \ + libboost-all-dev libssl-dev librocksdb-dev libsparsehash-dev \ + libsecp256k1-dev librange-v3-dev nlohmann-json3-dev libcereal-dev \ + libgtest-dev libcurl4-openssl-dev libjsoncpp-dev +``` + +#### **macOS (Homebrew):** + +```bash +brew install \ + cmake git boost openssl google-sparsehash secp256k1 \ + range-v3 nlohmann-json cereal googletest curl jsoncpp \ + bzip2 lz4 snappy zstd +``` + +Homebrew's latest `rocksdb` currently requires newer C++ headers than BlockSci's +supported C++17 build. Build the pinned RocksDB version into `$PREFIX` with the +unpackaged dependency script instead. +#### **Arch Linux:** + +> **Note:** Arch is not supported yet. Its rolling `rocksdb` (≥ 11) removed the +> `DB**` `Open` API that BlockSci still uses, so the library fails to compile. +> Use Ubuntu or macOS until BlockSci's rocksdb usage is modernized. + +```bash +sudo pacman -S --needed \ + base-devel cmake git boost openssl rocksdb sparsehash libsecp256k1 \ + range-v3 nlohmann-json cereal gtest curl jsoncpp +``` + +### Unpackaged dependencies + +A few small libraries are not available from apt or Homebrew. The helper script +builds them from source — pinned to the same revisions as the Nix flake — and +installs them into `$PREFIX`: + +```bash +./scripts/install-unpackaged-deps.sh "$PREFIX" +``` + +On macOS, include the pinned RocksDB build used by CI: + +```bash +BLOCKSCI_BUILD_ROCKSDB=1 ./scripts/install-unpackaged-deps.sh "$PREFIX" +``` + +This installs the header-only libraries `mpark/variant`, `wjfilesystem`, `mio`, +`dset`, `steinwurf/endian`, `clipp`, and Howard Hinnant's `date`, plus the +compiled libraries `libjson-rpc-cpp` and `bitcoin-api-cpp`. + +## Build libblocksci and the tools + +### For general OS + +```bash +cmake -B release \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX" +cmake --build release +cmake --install release + +``` + +### For macOS (Apple silicon) + +```bash +cmake -B release \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX;$(brew --prefix)" \ + -DOPENSSL_ROOT_DIR="$(brew --prefix openssl)" +cmake --build release +cmake --install release +``` + +`CMAKE_PREFIX_PATH` points CMake at the unpackaged dependencies installed above. +On macOS, append Homebrew's prefix so its packages are found, and point CMake at +the keg-only OpenSSL: +`-DCMAKE_PREFIX_PATH="$PREFIX;$(brew --prefix)" -DOPENSSL_ROOT_DIR="$(brew --prefix openssl)"`. + +## Build the Python bindings + +On Debian/Ubuntu the bindings also need `python3-dev` (Python headers) and, for +the step below, `python3-venv` — both split out from the base `python3` package +(other platforms' `python` already includes them). + +Install into a virtual environment so the build does not touch system Python: + +```bash +python3 -m venv "$PREFIX/venv" +source "$PREFIX/venv/bin/activate" +``` +`pip` reads `blockscipy/pyproject.toml`, builds the pybind11 module against the +installed C++ library, and pulls the Python runtime dependencies from PyPI: + +```bash +CMAKE_PREFIX_PATH="$PREFIX" pip install ./blockscipy +``` + +## Make the library findable at runtime + +The tools and the Python module load `libblocksci` at run time. Unless `$PREFIX` +is a standard system location, add its library directory to the loader path: + +```bash +# Linux +export LD_LIBRARY_PATH="$PREFIX/lib:$LD_LIBRARY_PATH" +# macOS +export DYLD_LIBRARY_PATH="$PREFIX/lib:$DYLD_LIBRARY_PATH" +``` + +## Verify + +```bash +"$PREFIX/bin/blocksci_parser" --help +python -c "import blocksci; print(blocksci.VERSION)" +``` diff --git a/docker/ubuntu.Dockerfile b/docker/ubuntu.Dockerfile new file mode 100644 index 00000000..81137d34 --- /dev/null +++ b/docker/ubuntu.Dockerfile @@ -0,0 +1,32 @@ +# Validates the non-nix build (build.md) on Ubuntu 24.04 — the GitHub CI image. +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake git \ + libboost-all-dev libssl-dev librocksdb-dev libsparsehash-dev \ + libsecp256k1-dev librange-v3-dev nlohmann-json3-dev libcereal-dev \ + libgtest-dev libcurl4-openssl-dev libjsoncpp-dev \ + python3 python3-dev python3-venv \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY . . + +ENV PREFIX=/src/install +RUN ./scripts/install-unpackaged-deps.sh "$PREFIX" + +RUN cmake -B release \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX" \ + && cmake --build release -j"$(nproc)" \ + && cmake --install release + +ENV LD_LIBRARY_PATH=/src/install/lib +RUN python3 -m venv "$PREFIX/venv" \ + && . "$PREFIX/venv/bin/activate" \ + && CMAKE_PREFIX_PATH="$PREFIX" pip install ./blockscipy \ + && "$PREFIX/bin/blocksci_parser" --help \ + && python -c "import blocksci; print(blocksci.VERSION)" diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 46071fef..816ae147 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -81,3 +81,9 @@ find_path(BITCOINAPI_INCLUDE_DIR bitcoinapi/bitcoinapi.h REQUIRED) add_library(bitcoinapi INTERFACE) target_include_directories(bitcoinapi SYSTEM INTERFACE ${BITCOINAPI_INCLUDE_DIR}) target_link_libraries(bitcoinapi INTERFACE ${BITCOINAPI_LIBRARY}) + +# bitcoin-api-cpp headers include ; give consumers jsoncpp's path. +find_path(JSONCPP_INCLUDE_DIR json/json.h PATH_SUFFIXES jsoncpp) +if(JSONCPP_INCLUDE_DIR) + target_include_directories(bitcoinapi SYSTEM INTERFACE ${JSONCPP_INCLUDE_DIR}) +endif() diff --git a/flake.lock b/flake.lock index 81272c25..5ea60b03 100644 --- a/flake.lock +++ b/flake.lock @@ -20,16 +20,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1779102034, - "narHash": "sha256-vZJZjLo513IeI8hjzHFc6TDezUd4uCE2Eq4SNO3DNNg=", + "lastModified": 1782375420, + "narHash": "sha256-wiPYmEuHbJvleW489n6+lamL7JSJg3pcKUYwURU9CkI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "687f05a9184cad4eaf905c48b63649e3a86f5433", + "rev": "4062d36ebeae843c750011eef6b61ec9a9dbc9a9", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-25.11", + "ref": "nixos-26.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a78937e3..acbb7187 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "BlockSci analysis tool for blockchains"; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; flake-utils.url = "github:numtide/flake-utils"; }; @@ -12,20 +12,14 @@ nixpkgs, flake-utils, }: - flake-utils.lib.eachSystem [ "x86_64-linux" ] ( + flake-utils.lib.eachDefaultSystem ( system: let pkgs = import nixpkgs { inherit system; overlays = [ (final: prev: { - libjson-rpc-cpp = prev.libjson-rpc-cpp.overrideAttrs (old: { - postPatch = (old.postPatch or "") + '' - substituteInPlace CMakeLists.txt \ - --replace-fail 'option(WITH_COVERAGE "Build with code coverage flags" ON)' \ - 'option(WITH_COVERAGE "Build with code coverage flags" OFF)' - ''; - }); + libjson-rpc-cpp = final.callPackage ./nix/libjson-rpc-cpp.nix {}; }) ]; }; @@ -38,6 +32,57 @@ pycryptodome requests ]; + pytestRegtest = pkgs.python3Packages.buildPythonPackage rec { + pname = "pytest-regtest"; + version = "2.5.1"; + pyproject = true; + + src = pkgs.fetchPypi { + pname = "pytest_regtest"; + inherit version; + hash = "sha256-hPd/GPpwKVN3Zi3pbXIdA7MtBVCcU+E2au7BHb7VbsA="; + }; + + build-system = with pkgs.python3Packages; [ + setuptools + ]; + + dependencies = with pkgs.python3Packages; [ + deepdiff + pytest + ]; + + # nixpkgs 26.05 currently ships deepdiff 8.x while pytest-regtest + # 2.5.1 declares deepdiff >= 9. BlockSci's regression tests exercise + # the plugin successfully with deepdiff 8.x, so relax this until + # nixpkgs updates deepdiff or we add a dedicated deepdiff 9 package. + pythonRelaxDeps = [ "deepdiff" ]; + + pythonImportsCheck = [ "pytest_regtest" ]; + + doCheck = false; + }; + pythonTestDeps = + ps: [ + ps.pytest + ps."pytest-benchmark" + pytestRegtest + ps.pytz + ps.tzlocal + ]; + knownPools = pkgs.fetchFromGitHub { + owner = "blockchain"; + repo = "Blockchain-Known-Pools"; + rev = "29ab27c844ebdb63110f8783f73b9decd4abc221"; + sha256 = "sha256-nFWprW6+PXDCIW/IjbAm9PNTMs+Clu42gCtfNnTU8QI="; + }; + pythonDevEnv = pkgs.python3.withPackages ( + ps: + [ + self.packages.${system}.blockscipy + ] + ++ pythonTestDeps ps + ); mkHeaderOnly = { pname, @@ -86,7 +131,6 @@ gtest nlohmann_json openssl - rocksdb secp256k1 sparsehash self.packages.${system}.bitcoin-api-cpp @@ -94,6 +138,7 @@ self.packages.${system}.endian self.packages.${system}.filesystem self.packages.${system}.mio + self.packages.${system}.rocksdb_8_9_1 ]; propagatedBuildInputs = with pkgs; [ @@ -122,6 +167,17 @@ dontUseCmakeConfigure = true; + preBuild = '' + parallel_level="''${NIX_BUILD_CORES:-1}" + + if [ "$parallel_level" -lt 1 ]; then + parallel_level=1 + fi + + export CMAKE_BUILD_PARALLEL_LEVEL="$parallel_level" + echo "CMAKE_BUILD_PARALLEL_LEVEL=$CMAKE_BUILD_PARALLEL_LEVEL" + ''; + buildInputs = with pkgs; [ howard-hinnant-date self.packages.${system}.default @@ -129,42 +185,46 @@ dependencies = pythonRuntimeDeps pkgs.python3Packages; + postInstall = '' + mkdir -p "$out/${pkgs.python3.sitePackages}/blocksci/Blockchain-Known-Pools" + cp ${knownPools}/pools.json \ + "$out/${pkgs.python3.sitePackages}/blocksci/Blockchain-Known-Pools/pools.json" + ''; + pythonImportsCheck = [ "blocksci" ]; doCheck = false; }; devShells.default = pkgs.mkShell { - inputsFrom = [ + packages = with pkgs; [ self.packages.${system}.default + pythonDevEnv + clang-tools + cmake + ninja + ruff ]; - nativeBuildInputs = - with pkgs; - [ - python3 - python3Packages.pip - python3Packages.scikit-build-core - clang-tools - ninja - ruff - ] - ++ pythonRuntimeDeps pkgs.python3Packages; + BLOCKSCI_POOLS_JSON = "${knownPools}/pools.json"; + CMAKE_PREFIX_PATH = "${self.packages.${system}.default}"; + }; - buildInputs = with pkgs; [ - python3Packages.pybind11 - howard-hinnant-date + # Initial flake CI intentionally covers BTC regtest only. BCH/LTC + # coverage can be added as follow-up checks once BTC remains stable. + checks.blockscipy-btc = pkgs.runCommand "blocksci-python-btc-tests" { + nativeBuildInputs = [ + self.packages.${system}.default + pythonDevEnv ]; - - shellHook = '' - export LOCAL_PIP="$PWD/.nix-pip" - export PYTHONPATH="$LOCAL_PIP/${pkgs.python3.sitePackages}:$PYTHONPATH" - export PATH="$LOCAL_PIP/bin:$PATH" - - export CMAKE_PREFIX_PATH="$PWD/.nix-install:$CMAKE_PREFIX_PATH" - export LD_LIBRARY_PATH="$PWD/.nix-install/lib64:$LD_LIBRARY_PATH" - ''; - }; + BLOCKSCI_POOLS_JSON = "${knownPools}/pools.json"; + } '' + cp -R ${self}/test "$TMPDIR/test" + chmod -R u+w "$TMPDIR/test" + cd "$TMPDIR/test/blockscipy" + python -m pytest --btc -q + touch "$out" + ''; packages.bitcoin-api-cpp = pkgs.stdenv.mkDerivation { pname = "bitcoin-api-cpp"; @@ -180,6 +240,11 @@ nativeBuildInputs = with pkgs; [ cmake ]; + + postPatch = '' + substituteInPlace CMakeLists.txt \ + --replace-fail "set(CMAKE_CXX_STANDARD 14)" "set(CMAKE_CXX_STANDARD 17)" + ''; buildInputs = with pkgs; [ curl @@ -191,6 +256,64 @@ ]; }; + packages.rocksdb_8_9_1 = pkgs.stdenv.mkDerivation { + pname = "rocksdb"; + version = "8.9.1"; + + src = pkgs.fetchFromGitHub { + owner = "facebook"; + repo = "rocksdb"; + rev = "v8.9.1"; + hash = "sha256-Pl7t4FVOvnORWFS+gjy2EEUQlPxjLukWW5I5gzCQwkI="; + }; + + nativeBuildInputs = with pkgs; [ + cmake + ninja + ]; + + propagatedBuildInputs = with pkgs; [ + bzip2 + lz4 + snappy + zlib + zstd + ]; + + postPatch = '' + # GCC 15 no longer accepts relying on transitive cstdint includes. + substituteInPlace include/rocksdb/rocksdb_namespace.h \ + --replace-fail '#pragma once' '#pragma once + + #include ' + ''; + + cmakeFlags = [ + "-DPORTABLE=1" + "-DWITH_JEMALLOC=0" + "-DWITH_JNI=0" + "-DWITH_BENCHMARK_TOOLS=0" + "-DWITH_TESTS=0" + "-DWITH_TOOLS=0" + "-DWITH_CORE_TOOLS=0" + "-DWITH_BZ2=1" + "-DWITH_LZ4=1" + "-DWITH_SNAPPY=1" + "-DWITH_ZLIB=1" + "-DWITH_ZSTD=1" + "-DWITH_GFLAGS=0" + "-DUSE_RTTI=1" + "-DFAIL_ON_WARNINGS=NO" + ]; + + postFixup = '' + if [ -f "$out"/lib/pkgconfig/rocksdb.pc ]; then + substituteInPlace "$out"/lib/pkgconfig/rocksdb.pc \ + --replace-fail '="''${prefix}//' '="/' + fi + ''; + }; + packages.dset = mkHeaderOnly { pname = "dset"; version = "unstable-2015-06-28"; diff --git a/nix.md b/nix.md index ed25c22e..46b9ac7a 100644 --- a/nix.md +++ b/nix.md @@ -1,55 +1,138 @@ -# Building BlockSci with Nix +# Building and Testing BlockSci with Nix -This project ships a Nix flake that provides: +This repository is Nix-first. The flake provides: -- Two package derivations (`packages.default` for libblocksci + tools, - `packages.blockscipy` for the Python bindings). -- A development shell (`devShells.default`) that reproduces the upstream - build manual with all native dependencies preinstalled, no system pollution. +- `packages.default`: libblocksci and command-line tools, including `blocksci_parser`. +- `packages.blockscipy`: Python bindings for BlockSci. +- `devShells.default`: a ready-to-use development/test shell with BlockSci, pytest, and benchmark dependencies. -The flake is pinned to `nixpkgs-25.11`. Supported systems: `x86_64-linux`. +The flake is pinned to `nixpkgs-25.11` and targets the four default systems +(`x86_64`/`aarch64` × Linux/macOS). -## Build libblocksci and the command-line tools +## Build packages +Build the C++ library and command-line tools: + +```bash +nix build -L +``` + +Build the Python bindings on memory-constrained machines: + +```bash +nix build .#blockscipy -L --cores 1 --max-jobs 1 +``` + +Build the Python bindings with more parallelism on machines with more RAM: + +```bash +nix build .#blockscipy -L --cores 4 --max-jobs 1 +``` + +`packages.blockscipy` forwards `NIX_BUILD_CORES` to CMake through +`CMAKE_BUILD_PARALLEL_LEVEL`, so `--cores` controls the internal +CMake/Ninja parallelism used to compile the Python extension. `--max-jobs` +controls how many Nix derivations may build at the same time. Use +`--max-jobs 1` when you want one build running at a time. + +## Run flake checks + +Run the default checks for the current system: + +```bash +nix flake check -L --cores 4 --max-jobs 1 +``` + +On machines with limited RAM, lower the CMake/Ninja parallelism: + +```bash +nix flake check -L --cores 1 --max-jobs 1 +``` + +The current flake check intentionally runs the BTC Python regression suite +only. BCH/LTC checks can be added later as separate follow-ups. + +## Development shell + +Enter the shell. The shell depends on the Nix-built Python bindings, so the +first invocation may compile `blockscipy` before opening the shell. + +```bash +nix develop -L --cores 4 --max-jobs 1 ``` -nix build + +On memory-constrained machines, use a serial build: + +```bash +nix develop -L --cores 1 --max-jobs 1 ``` -## Build the Python bindings +Inside the shell, Python imports the Nix-built `blocksci` package directly. No `pip install`, `LOCAL_PIP`, `.nix-pip`, or manual `PYTHONPATH` setup is required. + +Useful smoke checks: +```bash +python -c 'import blocksci; print(blocksci.__file__)' +which blocksci_parser ``` -nix build .#blockscipy + +Both should point into `/nix/store`. + +## Run Python regression tests + +```bash +nix develop -L --cores 4 --max-jobs 1 +cd test/blockscipy +python -m pytest --btc -q ``` -## Building in the development shell +Or run the same test command without entering an interactive shell: + +```bash +nix develop -L --cores 4 --max-jobs 1 --command sh -c 'cd test/blockscipy && python -m pytest --btc -q -W error::DeprecationWarning' +``` -This workflow builds the project step by step, self-contained under -`.nix-install/`, `.nix-pip/` and `release`. +The tests use synthetic Bitcoin regtest blocks from `test/files/btc/regtest/` and call `blocksci_parser` from the Nix package. -1. Enter the shell: +## Run benchmarks +```bash +nix develop -L --cores 4 --max-jobs 1 +cd test/benchmark +python -m pytest --btc -v --benchmark-autosave --benchmark-warmup=true --benchmark-warmup-iterations=1 --benchmark-sort=name --benchmark-columns=mean,median,max,rounds,iterations ``` -nix develop + +## Logs, cache, and rebuilds + +Nix is quiet on success, especially when outputs are already present in the +store. A command that prints no build output may still have passed. Check the +exit status with: + +```bash +echo $? ``` -2. Build and install libblocksci: +After `nix build`, the `result` symlink points to the built output: ```bash -mkdir release && cd release -cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=../.nix-install .. -make -make install -cd .. +readlink result ``` -3. Build the Python bindings: +Inspect logs from existing build outputs: +```bash +nix log .#blockscipy +nix log .#checks.$(nix eval --raw --impure --expr builtins.currentSystem).blockscipy-btc ``` -pip install --no-build-isolation --prefix=$LOCAL_PIP ./blockscipy + +Force a rebuild when you need fresh live output: + +```bash +nix build .#blockscipy -L --rebuild --cores 4 --max-jobs 1 +nix flake check -L --rebuild --cores 4 --max-jobs 1 ``` -`LOCAL_PIP`, `CMAKE_PREFIX_PATH` and `LD_LIBRARY_PATH` are exported by the -shellHook so the two halves find each other at build and run time. -`--no-build-isolation` makes pip use the `scikit-build-core` backend from the -shell instead of fetching it from PyPI. +## Known-pools data + +`Blockchain-Known-Pools` is fetched by Nix and exposed through `BLOCKSCI_POOLS_JSON`. +The project no longer requires that repository as a Git submodule. diff --git a/nix/libjson-rpc-cpp.nix b/nix/libjson-rpc-cpp.nix new file mode 100644 index 00000000..298f39af --- /dev/null +++ b/nix/libjson-rpc-cpp.nix @@ -0,0 +1,130 @@ +{ + lib, + stdenv, + fetchFromGitHub, + pkg-config, + cmake, + argtable, + catch2, + curl, + doxygen, + hiredis, + jsoncpp, + libmicrohttpd, + patchelf, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "libjson-rpc-cpp"; + version = "1.4.1"; + + src = fetchFromGitHub { + owner = "cinemast"; + repo = "libjson-rpc-cpp"; + sha256 = "sha256-YCCZN4y88AixQeo24pk6YHfSCsJz8jJ97Dg40KM08cQ="; + rev = "v${finalAttrs.version}"; + }; + + env.NIX_CFLAGS_COMPILE = "-I${catch2}/include/catch2"; + + nativeBuildInputs = [ + pkg-config + cmake + doxygen + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ + patchelf + ]; + + buildInputs = [ + argtable + catch2 + curl + hiredis + jsoncpp + libmicrohttpd + ]; + + postPatch = '' + for f in cmake/FindArgtable.cmake \ + src/stubgenerator/stubgenerator.cpp \ + src/stubgenerator/stubgeneratorfactory.cpp + do + sed -i -re 's/argtable2/argtable3/g' $f + done + + sed -i -re 's#MATCHES "jsoncpp"#MATCHES ".*/jsoncpp/json$"#g' cmake/FindJsoncpp.cmake + + substituteInPlace CMakeLists.txt \ + --replace-fail "cmake_minimum_required(VERSION 3.0)" "cmake_minimum_required(VERSION 3.10)" \ + --replace-fail "cmake_policy(SET CMP0042 OLD)" "" + + # jsoncpp 1.9.7 dropped char const*/String const& overloads in favor of std::string_view. + substituteInPlace cmake/CMakeCompilerSettings.cmake \ + --replace-fail "-std=c++11" "-std=c++17" + ''; + + preConfigure = '' + mkdir -p Build/Install + pushd Build + ''; + + # this hack is needed because the cmake scripts + # require write permission to absolute paths + configurePhase = '' + runHook preConfigure + cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd)/Install \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release \ + -DWITH_COVERAGE=OFF \ + ${lib.optionalString stdenv.hostPlatform.isDarwin "-DCMAKE_INSTALL_NAME_DIR=$out/lib -DCMAKE_INSTALL_RPATH=$out/lib"} + runHook postConfigure + ''; + + preInstall = + lib.optionalString stdenv.hostPlatform.isLinux '' + function fixRunPath { + p=$(patchelf --print-rpath $1) + q="$p:${ + lib.makeLibraryPath [ + jsoncpp + argtable + libmicrohttpd + curl + ] + }:$out/lib" + patchelf --set-rpath $q $1 + } + '' + + '' + mkdir -p $out + ''; + + postInstall = '' + sed -i -re "s#-([LI]).*/Build/Install(.*)#-\1$out\2#g" Install/lib*/pkgconfig/*.pc + '' + + lib.optionalString stdenv.hostPlatform.isLinux '' + for f in Install/lib*/*.so* $(find Install/bin -executable -type f); do + fixRunPath $f + done + '' + + '' + cp -r Install/* $out + ''; + + installPhase = '' + runHook preInstall + make install + runHook postInstall + ''; + + meta = { + description = "C++ framework for json-rpc (json remote procedure call)"; + mainProgram = "jsonrpcstub"; + homepage = "https://github.com/cinemast/libjson-rpc-cpp"; + license = lib.licenses.mit; + platforms = lib.platforms.linux ++ lib.platforms.darwin; + sourceProvenance = with lib.sourceTypes; [ binaryBytecode ]; + maintainers = with lib.maintainers; [ robertrichter ]; + }; +}) diff --git a/scripts/install-unpackaged-deps.sh b/scripts/install-unpackaged-deps.sh new file mode 100755 index 00000000..14bd50df --- /dev/null +++ b/scripts/install-unpackaged-deps.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# +# Install BlockSci's unpackaged C++ dependencies from source. +# +# These are the dependencies that are not reliably available from system +# package managers (apt/Homebrew). Everything else is a system package; see +# build.md. Versions are pinned to mirror flake.nix, the canonical build. +# +# Usage: +# scripts/install-unpackaged-deps.sh [PREFIX] +# +# PREFIX is where headers and libraries are installed; it defaults to +# $PWD/install (override via the first argument or the PREFIX env var). Pass +# the same PREFIX to BlockSci's CMake configure step via -DCMAKE_PREFIX_PATH. + +set -euo pipefail + +PREFIX="${1:-${PREFIX:-$PWD/install}}" +mkdir -p "$PREFIX" +PREFIX="$(cd "$PREFIX" && pwd)" +JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)" + +workdir="$(mktemp -d)" +trap 'rm -rf "$workdir"' EXIT +trap 'exit 130' INT TERM + +log() { + if [ -t 2 ]; then + printf '\n\033[1m==> %s\033[0m\n' "$*" >&2 + else + printf '\n==> %s\n' "$*" >&2 + fi +} + +# fetch OWNER/REPO REV -> prints the path to the checked-out tree +fetch() { + local slug="$1" rev="$2" dest="$workdir/${1##*/}" + git clone --quiet "https://github.com/$slug.git" "$dest" + git -C "$dest" -c advice.detachedHead=false checkout --quiet "$rev" + printf '%s\n' "$dest" +} + +mkdir -p "$PREFIX/include" + +# --- header-only libraries ------------------------------------------------- +# Each is copied so the layout matches the find_path() calls in +# external/CMakeLists.txt (e.g. , ). + +log "mpark/variant 1.3.0" +src="$(fetch mpark/variant v1.3.0)" +cp -R "$src/include/." "$PREFIX/include/" + +log "hkalodner/filesystem ()" +src="$(fetch hkalodner/filesystem 8cb505d6f51479645e670366df45951eeacb9fc4)" +cp -R "$src/include/." "$PREFIX/include/" + +log "hkalodner/mio ()" +src="$(fetch hkalodner/mio 9d2d5f397cad9ca6d2dfa31cd6716fdd98542862)" +cp -R "$src/include/." "$PREFIX/include/" + +log "wjakob/dset ()" +src="$(fetch wjakob/dset 7967ef0e6041cd9d73b9c7f614ab8ae92e9e587a)" +mkdir -p "$PREFIX/include/dset" +cp "$src/dset.h" "$PREFIX/include/dset/" + +log "steinwurf/endian 10.0.0 ()" +src="$(fetch steinwurf/endian 10.0.0)" +cp -R "$src/src/endian" "$PREFIX/include/" + +log "muellan/clipp 1.2.3 ()" +src="$(fetch muellan/clipp v1.2.3)" +cp "$src/include/clipp.h" "$PREFIX/include/" + +log "HowardHinnant/date 3.0.1 (, needed by the Python bindings)" +src="$(fetch HowardHinnant/date v3.0.1)" +cp -R "$src/include/." "$PREFIX/include/" + +# --- compiled libraries ---------------------------------------------------- +# Homebrew installs outside the default search paths on macOS; add its prefix +# when present (jsoncpp and curl come from the system package manager). +prefix_path="$PREFIX" +if command -v brew >/dev/null 2>&1; then + prefix_path="$PREFIX;$(brew --prefix)" +fi + +if [ "${BLOCKSCI_BUILD_ROCKSDB:-0}" = "1" ]; then + log "facebook/rocksdb 8.9.1" + src="$(fetch facebook/rocksdb v8.9.1)" + sed -i.bak \ + 's|#pragma once|#pragma once\ +\ +#include |' \ + "$src/include/rocksdb/rocksdb_namespace.h" + cmake -S "$src" -B "$src/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$prefix_path" \ + -DPORTABLE=1 \ + -DWITH_JEMALLOC=0 \ + -DWITH_JNI=0 \ + -DWITH_BENCHMARK_TOOLS=0 \ + -DWITH_TESTS=0 \ + -DWITH_TOOLS=0 \ + -DWITH_CORE_TOOLS=0 \ + -DWITH_BZ2=1 \ + -DWITH_LZ4=1 \ + -DWITH_SNAPPY=1 \ + -DWITH_ZLIB=1 \ + -DWITH_ZSTD=1 \ + -DWITH_GFLAGS=0 \ + -DUSE_RTTI=1 \ + -DFAIL_ON_WARNINGS=NO + cmake --build "$src/build" -j "$JOBS" + cmake --install "$src/build" + if [ -f "$PREFIX/lib/pkgconfig/rocksdb.pc" ]; then + sed -i.bak 's|="${prefix}//|="/|g' "$PREFIX/lib/pkgconfig/rocksdb.pc" + fi + prefix_path="$PREFIX;$prefix_path" +fi + +log "cinemast/libjson-rpc-cpp 1.4.1 (HTTP client only)" +src="$(fetch cinemast/libjson-rpc-cpp v1.4.1)" +for pat in 'cmake_minimum_required(VERSION 3.0)' 'cmake_policy(SET CMP0042 OLD)'; do + grep -qF "$pat" "$src/CMakeLists.txt" \ + || { echo "libjson-rpc-cpp: expected '$pat' not found" >&2; exit 1; } +done +sed -i.bak \ + -e 's/cmake_minimum_required(VERSION 3.0)/cmake_minimum_required(VERSION 3.10)/' \ + -e '/cmake_policy(SET CMP0042 OLD)/d' \ + "$src/CMakeLists.txt" +cmake -S "$src" -B "$src/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$prefix_path" \ + -DHTTP_SERVER=NO -DREDIS_SERVER=NO -DREDIS_CLIENT=NO \ + -DCOMPILE_TESTS=NO -DCOMPILE_STUBGEN=NO -DCOMPILE_EXAMPLES=NO \ + -DWITH_COVERAGE=NO +cmake --build "$src/build" -j "$JOBS" +cmake --install "$src/build" + +log "hkalodner/bitcoin-api-cpp" +src="$(fetch hkalodner/bitcoin-api-cpp 952bef34a7ad63f752982afcc6231013db7a19dd)" +# Drop the optional test suite: it pulls Boost (unit tests only) and breaks on +# newer Boost/CMake (Arch). We only need the library. +grep -qF 'FIND_PACKAGE(Boost COMPONENTS unit_test_framework)' "$src/CMakeLists.txt" \ + || { echo "bitcoin-api-cpp: expected Boost test block not found" >&2; exit 1; } +sed -i.bak '/FIND_PACKAGE(Boost COMPONENTS unit_test_framework)/,/ENDIF()/d' "$src/CMakeLists.txt" +cmake -S "$src" -B "$src/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$prefix_path" +cmake --build "$src/build" -j "$JOBS" +cmake --install "$src/build" + +log "Done. Installed into $PREFIX" +echo "Configure BlockSci with -DCMAKE_PREFIX_PATH=$PREFIX (see build.md)." diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cada5e7a..8243668b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -44,7 +44,7 @@ cxx_variadic_templates cxx_template_template_parameters ) -target_compile_features(blocksci PUBLIC ${BLOCKSCI_FEATURE_REQUIREMENTS}) +target_compile_features(blocksci PUBLIC cxx_std_17 ${BLOCKSCI_FEATURE_REQUIREMENTS}) target_compile_options(blocksci PRIVATE -Wall -Wextra -Wpedantic) diff --git a/src/internal/CMakeLists.txt b/src/internal/CMakeLists.txt index a6a9116c..b9a2408e 100644 --- a/src/internal/CMakeLists.txt +++ b/src/internal/CMakeLists.txt @@ -44,7 +44,7 @@ cxx_variadic_templates cxx_template_template_parameters ) -target_compile_features(blocksci_internal PUBLIC ${BLOCKSCI_FEATURE_REQUIREMENTS}) +target_compile_features(blocksci_internal PUBLIC cxx_std_17 ${BLOCKSCI_FEATURE_REQUIREMENTS}) target_compile_options(blocksci_internal PRIVATE -Wall -Wextra -Wpedantic) diff --git a/test/blockscipy/test_addresses.py b/test/blockscipy/test_addresses.py index f7539097..85d683fd 100644 --- a/test/blockscipy/test_addresses.py +++ b/test/blockscipy/test_addresses.py @@ -4,6 +4,13 @@ from util import Coin, sorted_tx_list +def test_address_type_str_repr(): + assert str(blocksci.address_type.pubkeyhash) == "Pay to pubkey hash" + assert repr(blocksci.address_type.pubkeyhash) == "address_type.pubkeyhash" + assert str(blocksci.address_type.scripthash) == "Pay to script hash" + assert repr(blocksci.address_type.scripthash) == "address_type.scripthash" + + def address_types(chain_name): if chain_name == "btc": return ["p2pkh", "p2sh", "p2wpkh", "p2wsh"] diff --git a/test/blockscipy/util.py b/test/blockscipy/util.py index 9b8d3bdb..91d7e0b7 100644 --- a/test/blockscipy/util.py +++ b/test/blockscipy/util.py @@ -6,7 +6,10 @@ def correct_timestamp(ts): tz = get_localzone() - ts_local = tz.localize(ts) + if hasattr(tz, "localize"): + ts_local = tz.localize(ts) + else: + ts_local = ts.replace(tzinfo=tz) return ts_local.astimezone(pytz.utc) diff --git a/test/requirements.txt b/test/requirements.txt index 45c9c2d8..05863878 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,7 @@ +# Test dependencies for non-Nix environments. +# Nix users should use `nix develop`; the flake provides these dependencies. pytest>=3.7.3 pytest-regtest>=1.3.1 tzlocal>=1.5.1 pytz>=2018.5 -pytest-benchmark>=3.2.3 \ No newline at end of file +pytest-benchmark>=3.2.3 diff --git a/tools/parser/address_state.hpp b/tools/parser/address_state.hpp index 461855b5..d0c8baec 100644 --- a/tools/parser/address_state.hpp +++ b/tools/parser/address_state.hpp @@ -36,10 +36,10 @@ template struct NonDudupAddressInfo { uint32_t addressNum; }; -template constexpr int startingCount = 0; -template <> constexpr int startingCount = 600'000'000; -template <> constexpr int startingCount = 100'000'000; -template <> constexpr int startingCount = 100'000'000; +template inline constexpr int startingCount = 0; +template <> inline constexpr int startingCount = 600'000'000; +template <> inline constexpr int startingCount = 100'000'000; +template <> inline constexpr int startingCount = 100'000'000; class AddressState { static constexpr auto AddressFalsePositiveRate = .05; diff --git a/tools/parser/block_processor.cpp b/tools/parser/block_processor.cpp index 188f05f2..0a15cf37 100644 --- a/tools/parser/block_processor.cpp +++ b/tools/parser/block_processor.cpp @@ -35,7 +35,6 @@ #include #include -#include #include #include diff --git a/tools/parser/chain_index.cpp b/tools/parser/chain_index.cpp index d20250e3..f387e125 100644 --- a/tools/parser/chain_index.cpp +++ b/tools/parser/chain_index.cpp @@ -19,7 +19,6 @@ #include #include -#include #include