From 8eeb3bbd2272ec8e67ebc67b811a930ba7a20632 Mon Sep 17 00:00:00 2001 From: Edil Medeiros Date: Thu, 4 Jun 2026 21:36:01 -0300 Subject: [PATCH 01/25] fix build on darwin using nix closes #1 --- flake.lock | 8 +- flake.nix | 30 ++++--- nix/libjson-rpc-cpp.nix | 130 +++++++++++++++++++++++++++++++ tools/parser/address_state.hpp | 8 +- tools/parser/block_processor.cpp | 1 - tools/parser/chain_index.cpp | 1 - 6 files changed, 157 insertions(+), 21 deletions(-) create mode 100644 nix/libjson-rpc-cpp.nix diff --git a/flake.lock b/flake.lock index 81272c25..ad4683c9 100644 --- a/flake.lock +++ b/flake.lock @@ -20,16 +20,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1779102034, - "narHash": "sha256-vZJZjLo513IeI8hjzHFc6TDezUd4uCE2Eq4SNO3DNNg=", + "lastModified": 1780453794, + "narHash": "sha256-bXMRa9VTsHSPXL4Cw8R6JJLQeY3Y/IP4+YJCYVmQ7FY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "687f05a9184cad4eaf905c48b63649e3a86f5433", + "rev": "6b316287bae2ee04c9b93c8c858d930fd07d7338", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-25.11", + "ref": "nixos-26.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a78937e3..e0ddedcc 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "BlockSci analysis tool for blockchains"; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; flake-utils.url = "github:numtide/flake-utils"; }; @@ -12,20 +12,14 @@ nixpkgs, flake-utils, }: - flake-utils.lib.eachSystem [ "x86_64-linux" ] ( + flake-utils.lib.eachDefaultSystem ( system: let pkgs = import nixpkgs { inherit system; overlays = [ (final: prev: { - libjson-rpc-cpp = prev.libjson-rpc-cpp.overrideAttrs (old: { - postPatch = (old.postPatch or "") + '' - substituteInPlace CMakeLists.txt \ - --replace-fail 'option(WITH_COVERAGE "Build with code coverage flags" ON)' \ - 'option(WITH_COVERAGE "Build with code coverage flags" OFF)' - ''; - }); + libjson-rpc-cpp = final.callPackage ./nix/libjson-rpc-cpp.nix {}; }) ]; }; @@ -162,8 +156,17 @@ export PATH="$LOCAL_PIP/bin:$PATH" export CMAKE_PREFIX_PATH="$PWD/.nix-install:$CMAKE_PREFIX_PATH" - export LD_LIBRARY_PATH="$PWD/.nix-install/lib64:$LD_LIBRARY_PATH" - ''; + '' + + ( + if pkgs.stdenv.isDarwin then + '' + export DYLD_LIBRARY_PATH="$PWD/.nix-install/lib:$DYLD_LIBRARY_PATH" + '' + else + '' + export LD_LIBRARY_PATH="$PWD/.nix-install/lib64:$LD_LIBRARY_PATH" + '' + ); }; packages.bitcoin-api-cpp = pkgs.stdenv.mkDerivation { @@ -180,6 +183,11 @@ nativeBuildInputs = with pkgs; [ cmake ]; + + postPatch = '' + substituteInPlace CMakeLists.txt \ + --replace-fail "set(CMAKE_CXX_STANDARD 14)" "set(CMAKE_CXX_STANDARD 17)" + ''; buildInputs = with pkgs; [ curl diff --git a/nix/libjson-rpc-cpp.nix b/nix/libjson-rpc-cpp.nix new file mode 100644 index 00000000..298f39af --- /dev/null +++ b/nix/libjson-rpc-cpp.nix @@ -0,0 +1,130 @@ +{ + lib, + stdenv, + fetchFromGitHub, + pkg-config, + cmake, + argtable, + catch2, + curl, + doxygen, + hiredis, + jsoncpp, + libmicrohttpd, + patchelf, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "libjson-rpc-cpp"; + version = "1.4.1"; + + src = fetchFromGitHub { + owner = "cinemast"; + repo = "libjson-rpc-cpp"; + sha256 = "sha256-YCCZN4y88AixQeo24pk6YHfSCsJz8jJ97Dg40KM08cQ="; + rev = "v${finalAttrs.version}"; + }; + + env.NIX_CFLAGS_COMPILE = "-I${catch2}/include/catch2"; + + nativeBuildInputs = [ + pkg-config + cmake + doxygen + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ + patchelf + ]; + + buildInputs = [ + argtable + catch2 + curl + hiredis + jsoncpp + libmicrohttpd + ]; + + postPatch = '' + for f in cmake/FindArgtable.cmake \ + src/stubgenerator/stubgenerator.cpp \ + src/stubgenerator/stubgeneratorfactory.cpp + do + sed -i -re 's/argtable2/argtable3/g' $f + done + + sed -i -re 's#MATCHES "jsoncpp"#MATCHES ".*/jsoncpp/json$"#g' cmake/FindJsoncpp.cmake + + substituteInPlace CMakeLists.txt \ + --replace-fail "cmake_minimum_required(VERSION 3.0)" "cmake_minimum_required(VERSION 3.10)" \ + --replace-fail "cmake_policy(SET CMP0042 OLD)" "" + + # jsoncpp 1.9.7 dropped char const*/String const& overloads in favor of std::string_view. + substituteInPlace cmake/CMakeCompilerSettings.cmake \ + --replace-fail "-std=c++11" "-std=c++17" + ''; + + preConfigure = '' + mkdir -p Build/Install + pushd Build + ''; + + # this hack is needed because the cmake scripts + # require write permission to absolute paths + configurePhase = '' + runHook preConfigure + cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd)/Install \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release \ + -DWITH_COVERAGE=OFF \ + ${lib.optionalString stdenv.hostPlatform.isDarwin "-DCMAKE_INSTALL_NAME_DIR=$out/lib -DCMAKE_INSTALL_RPATH=$out/lib"} + runHook postConfigure + ''; + + preInstall = + lib.optionalString stdenv.hostPlatform.isLinux '' + function fixRunPath { + p=$(patchelf --print-rpath $1) + q="$p:${ + lib.makeLibraryPath [ + jsoncpp + argtable + libmicrohttpd + curl + ] + }:$out/lib" + patchelf --set-rpath $q $1 + } + '' + + '' + mkdir -p $out + ''; + + postInstall = '' + sed -i -re "s#-([LI]).*/Build/Install(.*)#-\1$out\2#g" Install/lib*/pkgconfig/*.pc + '' + + lib.optionalString stdenv.hostPlatform.isLinux '' + for f in Install/lib*/*.so* $(find Install/bin -executable -type f); do + fixRunPath $f + done + '' + + '' + cp -r Install/* $out + ''; + + installPhase = '' + runHook preInstall + make install + runHook postInstall + ''; + + meta = { + description = "C++ framework for json-rpc (json remote procedure call)"; + mainProgram = "jsonrpcstub"; + homepage = "https://github.com/cinemast/libjson-rpc-cpp"; + license = lib.licenses.mit; + platforms = lib.platforms.linux ++ lib.platforms.darwin; + sourceProvenance = with lib.sourceTypes; [ binaryBytecode ]; + maintainers = with lib.maintainers; [ robertrichter ]; + }; +}) diff --git a/tools/parser/address_state.hpp b/tools/parser/address_state.hpp index 461855b5..d0c8baec 100644 --- a/tools/parser/address_state.hpp +++ b/tools/parser/address_state.hpp @@ -36,10 +36,10 @@ template struct NonDudupAddressInfo { uint32_t addressNum; }; -template constexpr int startingCount = 0; -template <> constexpr int startingCount = 600'000'000; -template <> constexpr int startingCount = 100'000'000; -template <> constexpr int startingCount = 100'000'000; +template inline constexpr int startingCount = 0; +template <> inline constexpr int startingCount = 600'000'000; +template <> inline constexpr int startingCount = 100'000'000; +template <> inline constexpr int startingCount = 100'000'000; class AddressState { static constexpr auto AddressFalsePositiveRate = .05; diff --git a/tools/parser/block_processor.cpp b/tools/parser/block_processor.cpp index 188f05f2..0a15cf37 100644 --- a/tools/parser/block_processor.cpp +++ b/tools/parser/block_processor.cpp @@ -35,7 +35,6 @@ #include #include -#include #include #include diff --git a/tools/parser/chain_index.cpp b/tools/parser/chain_index.cpp index d20250e3..f387e125 100644 --- a/tools/parser/chain_index.cpp +++ b/tools/parser/chain_index.cpp @@ -19,7 +19,6 @@ #include #include -#include #include From f5f7ecc8eccfa01bb2459eea8ac6db0fdddf5c14 Mon Sep 17 00:00:00 2001 From: Lucas Mendes <70616909+LucasMABF@users.noreply.github.com> Date: Mon, 22 Jun 2026 12:33:44 -0300 Subject: [PATCH 02/25] build: revert nixpkgs pin to 25.11 --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index ad4683c9..f5eb3c02 100644 --- a/flake.lock +++ b/flake.lock @@ -20,16 +20,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1780453794, - "narHash": "sha256-bXMRa9VTsHSPXL4Cw8R6JJLQeY3Y/IP4+YJCYVmQ7FY=", + "lastModified": 1781509190, + "narHash": "sha256-uJZs9Di8I6ciTp6jiojj0HzlNpBkud8ax5aT/O5aJkw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6b316287bae2ee04c9b93c8c858d930fd07d7338", + "rev": "d6df3513510aa548c83868fd22bfddd0a8c0a0d4", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-26.05", + "ref": "nixos-25.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index e0ddedcc..7e68bf0b 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "BlockSci analysis tool for blockchains"; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; flake-utils.url = "github:numtide/flake-utils"; }; From 7a3b32144704c5f2cef68c29f127eddd403bc29f Mon Sep 17 00:00:00 2001 From: Lucas Mendes <70616909+LucasMABF@users.noreply.github.com> Date: Mon, 22 Jun 2026 12:36:26 -0300 Subject: [PATCH 03/25] docs: note multi-platform flake and macOS linker path --- nix.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nix.md b/nix.md index ed25c22e..c55a749c 100644 --- a/nix.md +++ b/nix.md @@ -7,7 +7,8 @@ This project ships a Nix flake that provides: - A development shell (`devShells.default`) that reproduces the upstream build manual with all native dependencies preinstalled, no system pollution. -The flake is pinned to `nixpkgs-25.11`. Supported systems: `x86_64-linux`. +The flake is pinned to `nixpkgs-25.11` and targets the four default systems +(`x86_64`/`aarch64` × Linux/macOS). ## Build libblocksci and the command-line tools @@ -49,7 +50,8 @@ cd .. pip install --no-build-isolation --prefix=$LOCAL_PIP ./blockscipy ``` -`LOCAL_PIP`, `CMAKE_PREFIX_PATH` and `LD_LIBRARY_PATH` are exported by the +`LOCAL_PIP`, `CMAKE_PREFIX_PATH` and the dynamic-linker search path +(`LD_LIBRARY_PATH`, or `DYLD_LIBRARY_PATH` on macOS) are exported by the shellHook so the two halves find each other at build and run time. `--no-build-isolation` makes pip use the `scikit-build-core` backend from the shell instead of fetching it from PyPI. From 2104aaf6a667138cd78ac15b29091da8c7d8bd8c Mon Sep 17 00:00:00 2001 From: Lucas Mendes <70616909+LucasMABF@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:00:59 -0300 Subject: [PATCH 04/25] build: add non-nix build path docs for building with the system toolchain and package manager, a script for unpackaged deps, and an Ubuntu 24.04 Dockerfile. --- .dockerignore | 11 +++ .gitignore | 1 + build.md | 141 +++++++++++++++++++++++++++++ docker/ubuntu.Dockerfile | 32 +++++++ external/CMakeLists.txt | 6 ++ scripts/install-unpackaged-deps.sh | 123 +++++++++++++++++++++++++ 6 files changed, 314 insertions(+) create mode 100644 .dockerignore create mode 100644 build.md create mode 100644 docker/ubuntu.Dockerfile create mode 100755 scripts/install-unpackaged-deps.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..3d843db6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +release +install +build +result +**/.nix-install +**/.nix-pip +**/__pycache__ +**/*.egg-info +**/*.so +**/*.dylib +**/dist diff --git a/.gitignore b/.gitignore index 0a28194a..a400b07c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ release +install xcode debug build diff --git a/build.md b/build.md new file mode 100644 index 00000000..acd88aec --- /dev/null +++ b/build.md @@ -0,0 +1,141 @@ +# Building BlockSci from source + +This guide builds BlockSci on Linux or macOS using your system toolchain and +package manager, for environments where [Nix](nix.md) is not available. + +The Nix flake remains the canonical, reproducible build — it pins every +dependency and is what CI builds. Prefer it when you can; see [nix.md](nix.md). +This document is the supported manual path for everyone else. + +BlockSci is built in two halves: + +- `libblocksci` and the command-line tools (`blocksci_parser`, + `blocksci_clusterer`, …), built with CMake. +- `blockscipy`, the Python bindings, built on top of the installed C++ library + via `scikit-build-core`. + +## Requirements + +- A C++17 compiler (recent GCC or Clang) +- CMake ≥ 3.15 +- Python ≥ 3.11 with `pip` +- `git` + +Both halves and the from-source dependencies install into a single prefix so +they find each other. By default we use a disposable, project-local prefix — run +this from the repository root: + +```bash +export PREFIX="$PWD/install" +``` + +For a system-wide install, use `PREFIX=/usr/local` with `sudo`; it's on the +default loader path, so you can skip the `LD_LIBRARY_PATH` step at the end. + +## Dependencies + +BlockSci's dependencies fall into two groups: those your package manager +provides, and a handful that are not packaged and are built from source by a +helper script. + +### System packages + +#### **Debian / Ubuntu:** + +```bash +sudo apt install \ + build-essential cmake git \ + libboost-all-dev libssl-dev librocksdb-dev libsparsehash-dev \ + libsecp256k1-dev librange-v3-dev nlohmann-json3-dev libcereal-dev \ + libgtest-dev libcurl4-openssl-dev libjsoncpp-dev +``` + +#### **macOS (Homebrew):** + +```bash +brew install \ + cmake git boost openssl rocksdb google-sparsehash libsecp256k1 \ + range-v3 nlohmann-json cereal googletest curl jsoncpp +``` + +#### **Arch Linux:** + +> **Note:** Arch is not supported yet. Its rolling `rocksdb` (≥ 11) removed the +> `DB**` `Open` API that BlockSci still uses, so the library fails to compile. +> Use Ubuntu or macOS until BlockSci's rocksdb usage is modernized. + +```bash +sudo pacman -S --needed \ + base-devel cmake git boost openssl rocksdb sparsehash libsecp256k1 \ + range-v3 nlohmann-json cereal gtest curl jsoncpp +``` + +### Unpackaged dependencies + +A few small libraries are not available from apt or Homebrew. The helper script +builds them from source — pinned to the same revisions as the Nix flake — and +installs them into `$PREFIX`: + +```bash +./scripts/install-unpackaged-deps.sh "$PREFIX" +``` + +This installs the header-only libraries `mpark/variant`, `wjfilesystem`, `mio`, +`dset`, `steinwurf/endian`, `clipp`, and Howard Hinnant's `date`, plus the +compiled libraries `libjson-rpc-cpp` and `bitcoin-api-cpp`. + +## Build libblocksci and the tools + +```bash +cmake -B release \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX" +cmake --build release +cmake --install release +``` + +`CMAKE_PREFIX_PATH` points CMake at the unpackaged dependencies installed above. +On macOS, append Homebrew's prefix so its packages are found, and point CMake at +the keg-only OpenSSL: +`-DCMAKE_PREFIX_PATH="$PREFIX;$(brew --prefix)" -DOPENSSL_ROOT_DIR="$(brew --prefix openssl)"`. + +## Build the Python bindings + +On Debian/Ubuntu the bindings also need `python3-dev` (Python headers) and, for +the step below, `python3-venv` — both split out from the base `python3` package +(other platforms' `python` already includes them). + +Install into a virtual environment so the build does not touch system Python: + +```bash +python3 -m venv "$PREFIX/venv" +source "$PREFIX/venv/bin/activate" +``` + +`pip` reads `blockscipy/pyproject.toml`, builds the pybind11 module against the +installed C++ library, and pulls the Python runtime dependencies from PyPI: + +```bash +CMAKE_PREFIX_PATH="$PREFIX" pip install ./blockscipy +``` + +## Make the library findable at runtime + +The tools and the Python module load `libblocksci` at run time. Unless `$PREFIX` +is a standard system location, add its library directory to the loader path: + +```bash +# Linux +export LD_LIBRARY_PATH="$PREFIX/lib:$LD_LIBRARY_PATH" +# macOS +export DYLD_LIBRARY_PATH="$PREFIX/lib:$DYLD_LIBRARY_PATH" +``` + +## Verify + +```bash +"$PREFIX/bin/blocksci_parser" --help +python -c "import blocksci; print(blocksci.VERSION)" +``` diff --git a/docker/ubuntu.Dockerfile b/docker/ubuntu.Dockerfile new file mode 100644 index 00000000..81137d34 --- /dev/null +++ b/docker/ubuntu.Dockerfile @@ -0,0 +1,32 @@ +# Validates the non-nix build (build.md) on Ubuntu 24.04 — the GitHub CI image. +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake git \ + libboost-all-dev libssl-dev librocksdb-dev libsparsehash-dev \ + libsecp256k1-dev librange-v3-dev nlohmann-json3-dev libcereal-dev \ + libgtest-dev libcurl4-openssl-dev libjsoncpp-dev \ + python3 python3-dev python3-venv \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY . . + +ENV PREFIX=/src/install +RUN ./scripts/install-unpackaged-deps.sh "$PREFIX" + +RUN cmake -B release \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX" \ + && cmake --build release -j"$(nproc)" \ + && cmake --install release + +ENV LD_LIBRARY_PATH=/src/install/lib +RUN python3 -m venv "$PREFIX/venv" \ + && . "$PREFIX/venv/bin/activate" \ + && CMAKE_PREFIX_PATH="$PREFIX" pip install ./blockscipy \ + && "$PREFIX/bin/blocksci_parser" --help \ + && python -c "import blocksci; print(blocksci.VERSION)" diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 46071fef..816ae147 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -81,3 +81,9 @@ find_path(BITCOINAPI_INCLUDE_DIR bitcoinapi/bitcoinapi.h REQUIRED) add_library(bitcoinapi INTERFACE) target_include_directories(bitcoinapi SYSTEM INTERFACE ${BITCOINAPI_INCLUDE_DIR}) target_link_libraries(bitcoinapi INTERFACE ${BITCOINAPI_LIBRARY}) + +# bitcoin-api-cpp headers include ; give consumers jsoncpp's path. +find_path(JSONCPP_INCLUDE_DIR json/json.h PATH_SUFFIXES jsoncpp) +if(JSONCPP_INCLUDE_DIR) + target_include_directories(bitcoinapi SYSTEM INTERFACE ${JSONCPP_INCLUDE_DIR}) +endif() diff --git a/scripts/install-unpackaged-deps.sh b/scripts/install-unpackaged-deps.sh new file mode 100755 index 00000000..9d99c2cc --- /dev/null +++ b/scripts/install-unpackaged-deps.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# Install BlockSci's unpackaged C++ dependencies from source. +# +# These are the dependencies that are not reliably available from system +# package managers (apt/Homebrew). Everything else is a system package; see +# build.md. Versions are pinned to mirror flake.nix, the canonical build. +# +# Usage: +# scripts/install-unpackaged-deps.sh [PREFIX] +# +# PREFIX is where headers and libraries are installed; it defaults to +# $PWD/install (override via the first argument or the PREFIX env var). Pass +# the same PREFIX to BlockSci's CMake configure step via -DCMAKE_PREFIX_PATH. + +set -euo pipefail + +PREFIX="${1:-${PREFIX:-$PWD/install}}" +mkdir -p "$PREFIX" +PREFIX="$(cd "$PREFIX" && pwd)" +JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)" + +workdir="$(mktemp -d)" +trap 'rm -rf "$workdir"' EXIT +trap 'exit 130' INT TERM + +log() { + if [ -t 2 ]; then + printf '\n\033[1m==> %s\033[0m\n' "$*" >&2 + else + printf '\n==> %s\n' "$*" >&2 + fi +} + +# fetch OWNER/REPO REV -> prints the path to the checked-out tree +fetch() { + local slug="$1" rev="$2" dest="$workdir/${1##*/}" + git clone --quiet "https://github.com/$slug.git" "$dest" + git -C "$dest" -c advice.detachedHead=false checkout --quiet "$rev" + printf '%s\n' "$dest" +} + +mkdir -p "$PREFIX/include" + +# --- header-only libraries ------------------------------------------------- +# Each is copied so the layout matches the find_path() calls in +# external/CMakeLists.txt (e.g. , ). + +log "mpark/variant 1.3.0" +src="$(fetch mpark/variant v1.3.0)" +cp -R "$src/include/." "$PREFIX/include/" + +log "hkalodner/filesystem ()" +src="$(fetch hkalodner/filesystem 8cb505d6f51479645e670366df45951eeacb9fc4)" +cp -R "$src/include/." "$PREFIX/include/" + +log "hkalodner/mio ()" +src="$(fetch hkalodner/mio 9d2d5f397cad9ca6d2dfa31cd6716fdd98542862)" +cp -R "$src/include/." "$PREFIX/include/" + +log "wjakob/dset ()" +src="$(fetch wjakob/dset 7967ef0e6041cd9d73b9c7f614ab8ae92e9e587a)" +mkdir -p "$PREFIX/include/dset" +cp "$src/dset.h" "$PREFIX/include/dset/" + +log "steinwurf/endian 10.0.0 ()" +src="$(fetch steinwurf/endian 10.0.0)" +cp -R "$src/src/endian" "$PREFIX/include/" + +log "muellan/clipp 1.2.3 ()" +src="$(fetch muellan/clipp v1.2.3)" +cp "$src/include/clipp.h" "$PREFIX/include/" + +log "HowardHinnant/date 3.0.1 (, needed by the Python bindings)" +src="$(fetch HowardHinnant/date v3.0.1)" +cp -R "$src/include/." "$PREFIX/include/" + +# --- compiled libraries ---------------------------------------------------- +# Homebrew installs outside the default search paths on macOS; add its prefix +# when present (jsoncpp and curl come from the system package manager). +prefix_path="$PREFIX" +if command -v brew >/dev/null 2>&1; then + prefix_path="$PREFIX;$(brew --prefix)" +fi + +log "cinemast/libjson-rpc-cpp 1.4.1 (HTTP client only)" +src="$(fetch cinemast/libjson-rpc-cpp v1.4.1)" +for pat in 'cmake_minimum_required(VERSION 3.0)' 'cmake_policy(SET CMP0042 OLD)'; do + grep -qF "$pat" "$src/CMakeLists.txt" \ + || { echo "libjson-rpc-cpp: expected '$pat' not found" >&2; exit 1; } +done +sed -i.bak \ + -e 's/cmake_minimum_required(VERSION 3.0)/cmake_minimum_required(VERSION 3.10)/' \ + -e '/cmake_policy(SET CMP0042 OLD)/d' \ + "$src/CMakeLists.txt" +cmake -S "$src" -B "$src/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$prefix_path" \ + -DHTTP_SERVER=NO -DREDIS_SERVER=NO -DREDIS_CLIENT=NO \ + -DCOMPILE_TESTS=NO -DCOMPILE_STUBGEN=NO -DCOMPILE_EXAMPLES=NO \ + -DWITH_COVERAGE=NO +cmake --build "$src/build" -j "$JOBS" +cmake --install "$src/build" + +log "hkalodner/bitcoin-api-cpp" +src="$(fetch hkalodner/bitcoin-api-cpp 952bef34a7ad63f752982afcc6231013db7a19dd)" +# Drop the optional test suite: it pulls Boost (unit tests only) and breaks on +# newer Boost/CMake (Arch). We only need the library. +grep -qF 'FIND_PACKAGE(Boost COMPONENTS unit_test_framework)' "$src/CMakeLists.txt" \ + || { echo "bitcoin-api-cpp: expected Boost test block not found" >&2; exit 1; } +sed -i.bak '/FIND_PACKAGE(Boost COMPONENTS unit_test_framework)/,/ENDIF()/d' "$src/CMakeLists.txt" +cmake -S "$src" -B "$src/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$prefix_path" +cmake --build "$src/build" -j "$JOBS" +cmake --install "$src/build" + +log "Done. Installed into $PREFIX" +echo "Configure BlockSci with -DCMAKE_PREFIX_PATH=$PREFIX (see build.md)." From d699460cce0d6980eddc7f355f3a1506cfaf07fa Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:15:30 -0300 Subject: [PATCH 05/25] fix(blockscipy): support modern range-v3 bindings --- blockscipy/src/blocksci_range_type.cpp | 2 +- blockscipy/src/blocksci_type_converter.cpp | 10 ++++---- blockscipy/src/blocksci_type_converter.hpp | 26 ++++++++++----------- blockscipy/src/proxy/range.hpp | 4 ++-- blockscipy/src/python_range_conversion.cpp | 27 +++++++++++----------- 5 files changed, 35 insertions(+), 34 deletions(-) diff --git a/blockscipy/src/blocksci_range_type.cpp b/blockscipy/src/blocksci_range_type.cpp index 3847a405..74c95d5a 100644 --- a/blockscipy/src/blocksci_range_type.cpp +++ b/blockscipy/src/blocksci_range_type.cpp @@ -29,7 +29,7 @@ struct BlocksciRangeTypeObjectCaster { struct BlockSciRangeToIterator { template BlocksciIteratorType operator()(T &&t) { - return BlocksciIteratorType{RawIterator>{std::forward(t)}}; + return BlocksciIteratorType{RawIterator>{std::forward(t)}}; } }; diff --git a/blockscipy/src/blocksci_type_converter.cpp b/blockscipy/src/blocksci_type_converter.cpp index 700ba159..e6939bc3 100644 --- a/blockscipy/src/blocksci_type_converter.cpp +++ b/blockscipy/src/blocksci_type_converter.cpp @@ -20,18 +20,18 @@ #include -blocksci::AnyScript BlockSciTypeConverter::operator()(const blocksci::Address &address) { +blocksci::AnyScript BlockSciTypeConverter::operator()(const blocksci::Address &address) const { return address.getScript(); } -RawRange BlockSciTypeConverter::operator()(const blocksci::InputRange &val) { +RawRange BlockSciTypeConverter::operator()(const blocksci::InputRange &val) const { return ranges::any_view{val}; } -RawRange BlockSciTypeConverter::operator()(const blocksci::OutputRange &val) { +RawRange BlockSciTypeConverter::operator()(const blocksci::OutputRange &val) const { return ranges::any_view{val}; } -RawRange BlockSciTypeConverter::operator()(const blocksci::BlockRange &val) { +RawRange BlockSciTypeConverter::operator()(const blocksci::BlockRange &val) const { return ranges::any_view{val}; -} \ No newline at end of file +} diff --git a/blockscipy/src/blocksci_type_converter.hpp b/blockscipy/src/blocksci_type_converter.hpp index bbecf838..3d122ff9 100644 --- a/blockscipy/src/blocksci_type_converter.hpp +++ b/blockscipy/src/blocksci_type_converter.hpp @@ -22,39 +22,39 @@ struct BlockSciTypeConverter { - template T operator()(const T &t) { + template T operator()(const T &t) const { return t; } - blocksci::AnyScript operator()(const blocksci::Address &address); + blocksci::AnyScript operator()(const blocksci::Address &address) const; - int64_t operator()(uint16_t val) { + int64_t operator()(uint16_t val) const { return static_cast(val); } - int64_t operator()(int16_t val) { + int64_t operator()(int16_t val) const { return static_cast(val); } - int64_t operator()(uint32_t val) { + int64_t operator()(uint32_t val) const { return static_cast(val); } - int64_t operator()(int32_t val) { + int64_t operator()(int32_t val) const { return static_cast(val); } // Potential overflow - int64_t operator()(uint64_t val) { + int64_t operator()(uint64_t val) const { return static_cast(val); } - RawRange operator()(const blocksci::InputRange &val); - RawRange operator()(const blocksci::OutputRange &val); - RawRange operator()(const blocksci::BlockRange &val); + RawRange operator()(const blocksci::InputRange &val) const; + RawRange operator()(const blocksci::OutputRange &val) const; + RawRange operator()(const blocksci::BlockRange &val) const; template - auto operator()(const ranges::optional &val) -> ranges::optionaloperator()(*val))> { + auto operator()(const ranges::optional &val) const -> ranges::optionaloperator()(*val))> { if (val) { return this->operator()(*val); } else { @@ -62,11 +62,11 @@ struct BlockSciTypeConverter { } } - template auto operator()(Iterator &val) -> Iteratoroperator()(*val))> { + template auto operator()(Iterator &val) const -> Iteratoroperator()(*val))> { return val | ranges::views::transform(this); } - template auto operator()(Range &val) -> Rangeoperator()(*val))> { + template auto operator()(Range &val) const -> Rangeoperator()(*val))> { return val | ranges::views::transform(this); } }; diff --git a/blockscipy/src/proxy/range.hpp b/blockscipy/src/proxy/range.hpp index 162d3b73..4d75ef8e 100644 --- a/blockscipy/src/proxy/range.hpp +++ b/blockscipy/src/proxy/range.hpp @@ -97,8 +97,8 @@ template void setupRangesProxy(AllProxyClasses if (!slice.compute(chainSize, &start, &stop, &step, &slicelength)) throw pybind11::error_already_set(); - auto subset = range[{static_cast>>(start), - static_cast>>(stop)}]; + auto subset = range[{static_cast>>(start), + static_cast>>(stop)}]; return subset | ranges::views::stride(step); }); }, diff --git a/blockscipy/src/python_range_conversion.cpp b/blockscipy/src/python_range_conversion.cpp index 2753c976..19ea78d0 100644 --- a/blockscipy/src/python_range_conversion.cpp +++ b/blockscipy/src/python_range_conversion.cpp @@ -37,48 +37,49 @@ namespace py = pybind11; struct NumpyConverter { - template T operator()(const T &val) { + template T operator()(const T &val) const { return val; } - NumpyDatetime operator()(const std::chrono::system_clock::time_point &val) { + NumpyDatetime operator()(const std::chrono::system_clock::time_point &val) const { return {std::chrono::duration_cast(val.time_since_epoch()).count()}; } - std::array operator()(const blocksci::uint256 &val) { + std::array operator()(const blocksci::uint256 &val) const { auto hexStr = val.GetHex(); std::array ret; std::copy_n(hexStr.begin(), 64, ret.begin()); return ret; } - std::array operator()(const blocksci::uint160 &val) { + std::array operator()(const blocksci::uint160 &val) const { auto hexStr = val.GetHex(); std::array ret; std::copy_n(hexStr.begin(), 40, ret.begin()); return ret; } - NumpyBool operator()(const bool &val) { + NumpyBool operator()(const bool &val) const { return {val}; } }; template -pybind11::array_t>()))> +pybind11::array_t>()))> convertRandomSizedNumpy(T &&t) { auto numpy_converted = ranges::views::transform(std::move(t), NumpyConverter{}); auto rangeSize = static_cast(ranges::size(numpy_converted)); - pybind11::array_t> ret{rangeSize}; + pybind11::array_t> ret{ + static_cast(rangeSize)}; auto retPtr = ret.mutable_data(); ranges::copy(numpy_converted, retPtr); return ret; } template -pybind11::array_t>()))> convertInputNumpy(T &&t) { +pybind11::array_t>()))> convertInputNumpy(T &&t) { auto ret = ranges::to(ranges::views::transform(std::move(t), NumpyConverter{})); - return pybind11::array_t{ret.size(), ret.data()}; + return pybind11::array_t{static_cast(ret.size()), ret.data()}; } template py::list convertRandomSizedPy(T &&t) { @@ -101,15 +102,15 @@ template py::list convertInputPy(T &&t) { } template -Iterator>()))> convertInputBlockSci(T &&t) { - return ranges::any_view>()))>{ +Iterator>()))> convertInputBlockSci(T &&t) { + return ranges::any_view>()))>{ ranges::views::transform(std::forward(t), BlockSciTypeConverter{})}; } template -Range>()))> +Range>()))> convertRandomSizedBlockSci(T &&t) { - return ranges::any_view>())), + return ranges::any_view>())), random_access_sized>{ranges::views::transform(std::forward(t), BlockSciTypeConverter{})}; } From 7980aa277b308f4e59431e660194703c37fbd2a7 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:16:03 -0300 Subject: [PATCH 06/25] fix(blockscipy): restore address type formatting --- blockscipy/blocksci/__init__.py | 42 +++++++++++++++++++++++++++++++ test/blockscipy/test_addresses.py | 7 ++++++ 2 files changed, 49 insertions(+) diff --git a/blockscipy/blocksci/__init__.py b/blockscipy/blocksci/__init__.py index 809786f2..4299c1f0 100644 --- a/blockscipy/blocksci/__init__.py +++ b/blockscipy/blocksci/__init__.py @@ -39,6 +39,48 @@ sys.modules["blocksci.heuristics"] = heuristics sys.modules["blocksci.heuristics.change"] = heuristics.change +# pybind11 3 gives enums Python-native ``Enum`` formatting by default +# (``address_type.pubkeyhash`` / ````). +# BlockSci's public API and regression fixtures historically exposed a +# human-readable ``str()`` and a compact qualified-name ``repr()``. +_ADDRESS_TYPE_LABELS = { + address_type.pubkey: "Pay to pubkey", + address_type.pubkeyhash: "Pay to pubkey hash", + address_type.scripthash: "Pay to script hash", + address_type.multisig: "Multisig", + address_type.multisig_pubkey: "Multisig Public Key", + address_type.nonstandard: "Nonstandard", + address_type.nulldata: "Null data", + address_type.witness_pubkeyhash: "Pay to witness pubkey hash", + address_type.witness_scripthash: "Pay to witness script hash", + address_type.witness_unknown: "Pay to witness unknown", +} + +_ADDRESS_TYPE_REPR_NAMES = { + address_type.pubkey: "pubkey", + address_type.pubkeyhash: "pubkeyhash", + address_type.scripthash: "scripthash", + address_type.multisig: "multisig", + address_type.multisig_pubkey: "multisig_pubkey", + address_type.nonstandard: "nonstandard", + address_type.nulldata: "nulldata", + address_type.witness_pubkeyhash: "witness_pubkeyhash", + address_type.witness_scripthash: "witness_scripthash", + address_type.witness_unknown: "witness_unknown", +} + + +def _address_type_str(value): + return _ADDRESS_TYPE_LABELS.get(value, "Unknown Address Type") + + +def _address_type_repr(value): + return f"address_type.{_ADDRESS_TYPE_REPR_NAMES.get(value, 'unknown')}" + + +address_type.__str__ = _address_type_str +address_type.__repr__ = _address_type_repr + class _NoDefault: def __repr__(self): diff --git a/test/blockscipy/test_addresses.py b/test/blockscipy/test_addresses.py index f7539097..85d683fd 100644 --- a/test/blockscipy/test_addresses.py +++ b/test/blockscipy/test_addresses.py @@ -4,6 +4,13 @@ from util import Coin, sorted_tx_list +def test_address_type_str_repr(): + assert str(blocksci.address_type.pubkeyhash) == "Pay to pubkey hash" + assert repr(blocksci.address_type.pubkeyhash) == "address_type.pubkeyhash" + assert str(blocksci.address_type.scripthash) == "Pay to script hash" + assert repr(blocksci.address_type.scripthash) == "address_type.scripthash" + + def address_types(chain_name): if chain_name == "btc": return ["p2pkh", "p2sh", "p2wpkh", "p2wsh"] From fb5e4129788fc1491aef00740d6bfb373086edaa Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:16:15 -0300 Subject: [PATCH 07/25] fix(tests): support zoneinfo timezones --- test/blockscipy/util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/blockscipy/util.py b/test/blockscipy/util.py index 9b8d3bdb..91d7e0b7 100644 --- a/test/blockscipy/util.py +++ b/test/blockscipy/util.py @@ -6,7 +6,10 @@ def correct_timestamp(ts): tz = get_localzone() - ts_local = tz.localize(ts) + if hasattr(tz, "localize"): + ts_local = tz.localize(ts) + else: + ts_local = ts.replace(tzinfo=tz) return ts_local.astimezone(pytz.utc) From 0bcebf65b3a996538ccb0edea5e3fc68b8ab14e7 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:16:44 -0300 Subject: [PATCH 08/25] fix(blockscipy): avoid fork warnings on modern macos --- blockscipy/blocksci/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/blockscipy/blocksci/__init__.py b/blockscipy/blocksci/__init__.py index 4299c1f0..aabe03ad 100644 --- a/blockscipy/blocksci/__init__.py +++ b/blockscipy/blocksci/__init__.py @@ -19,10 +19,10 @@ from functools import reduce import dateparser +import multiprocess as mp import pandas as pd import psutil from dateutil.relativedelta import relativedelta -from multiprocess import Pool from ._blocksci import * from ._blocksci import _traverse @@ -90,6 +90,15 @@ def __repr__(self): MISSING_PARAM = _NoDefault() +def _multiprocess_context(): + # Python 3.12+ warns when forking a multithreaded process. Pytest and + # native extension imports can start helper threads, so prefer spawn on + # Darwin where fork-without-exec is particularly fragile. + if sys.platform == "darwin" and sys.version_info >= (3, 12): + return mp.get_context("spawn") + return mp.get_context() + + # Alert the user if the disk space is getting full disk_info = os.statvfs("/") free_space = (disk_info.f_frsize * disk_info.f_bavail) // (1024**3) @@ -138,7 +147,7 @@ def real_map_func(input): file.seek(0) return file - with Pool(cpu_count - 1) as p: + with _multiprocess_context().Pool(cpu_count - 1) as p: results_future = p.map_async(real_map_func, segments[1:]) first = map_func(chain[raw_segments[0][0] : raw_segments[0][1]]) results = results_future.get() From 820060ba345dc0b6267ec952d1e92810c1ee1cf6 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:17:00 -0300 Subject: [PATCH 09/25] fix(blockscipy): make miner labels optional --- blockscipy/blocksci/__init__.py | 37 ++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/blockscipy/blocksci/__init__.py b/blockscipy/blocksci/__init__.py index aabe03ad..485b1225 100644 --- a/blockscipy/blocksci/__init__.py +++ b/blockscipy/blocksci/__init__.py @@ -916,6 +916,21 @@ class DummyClass: loaderDirectory = os.path.dirname(os.path.abspath(inspect.getsourcefile(DummyClass))) +def _load_pool_data(): + import json + + pools_path = os.environ.get( + "BLOCKSCI_POOLS_JSON", + os.path.join(loaderDirectory, "Blockchain-Known-Pools", "pools.json"), + ) + try: + with open(pools_path) as f: + return json.load(f) + except FileNotFoundError: + logging.warning("BlockSci miner labels disabled: pools.json not found at %s", pools_path) + return {"payout_addresses": {}, "coinbase_tags": {}} + + def get_miner(block) -> str: """ Get the miner of the block based on the text in the coinbase transaction @@ -925,20 +940,18 @@ def get_miner(block) -> str: global pool_data global coinbase_tag_re if first_miner_run: - import json - - with open(loaderDirectory + "/Blockchain-Known-Pools/pools.json") as f: - pool_data = json.load(f) - addresses = [block._access.address_from_string(addr_string) for addr_string in pool_data["payout_addresses"]] - tagged_addresses = { - pointer: pool_data["payout_addresses"][address] - for address in addresses - if address in pool_data["payout_addresses"] - } - coinbase_tag_re = re.compile("|".join(map(re.escape, pool_data["coinbase_tags"]))) + pool_data = _load_pool_data() + tagged_addresses = {} + for addr_string, pool_info in pool_data.get("payout_addresses", {}).items(): + try: + tagged_addresses[block._access.address_from_string(addr_string)] = pool_info + except Exception: + logging.debug("Skipping invalid miner payout address %s", addr_string, exc_info=True) + coinbase_tags = pool_data.get("coinbase_tags", {}) + coinbase_tag_re = re.compile("|".join(map(re.escape, coinbase_tags))) if coinbase_tags else None first_miner_run = False coinbase = block.coinbase_param.decode("utf_8", "replace") - tag_matches = re.findall(coinbase_tag_re, coinbase) + tag_matches = re.findall(coinbase_tag_re, coinbase) if coinbase_tag_re is not None else [] if tag_matches: return pool_data["coinbase_tags"][tag_matches[0]]["name"] for txout in block.coinbase_tx.outs: From c5c1dcaa2d45ded17d3571b7f7c12b836aa85312 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:17:45 -0300 Subject: [PATCH 10/25] chore(nix): fetch miner labels without submodule --- .gitmodules | 3 --- blockscipy/blocksci/Blockchain-Known-Pools | 1 - flake.nix | 12 ++++++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) delete mode 160000 blockscipy/blocksci/Blockchain-Known-Pools diff --git a/.gitmodules b/.gitmodules index bebab775..40889ddd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "Notebooks/blocksci/Blockchain-Known-Pools"] - path = blockscipy/blocksci/Blockchain-Known-Pools - url = https://github.com/blockchain/Blockchain-Known-Pools [submodule "test/testchain-generator"] path = test/testchain-generator url = https://github.com/citp/testchain-generator diff --git a/blockscipy/blocksci/Blockchain-Known-Pools b/blockscipy/blocksci/Blockchain-Known-Pools deleted file mode 160000 index 29ab27c8..00000000 --- a/blockscipy/blocksci/Blockchain-Known-Pools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 29ab27c844ebdb63110f8783f73b9decd4abc221 diff --git a/flake.nix b/flake.nix index 7e68bf0b..2a4cfc91 100644 --- a/flake.nix +++ b/flake.nix @@ -32,6 +32,12 @@ pycryptodome requests ]; + knownPools = pkgs.fetchFromGitHub { + owner = "blockchain"; + repo = "Blockchain-Known-Pools"; + rev = "29ab27c844ebdb63110f8783f73b9decd4abc221"; + sha256 = "sha256-nFWprW6+PXDCIW/IjbAm9PNTMs+Clu42gCtfNnTU8QI="; + }; mkHeaderOnly = { pname, @@ -123,6 +129,12 @@ dependencies = pythonRuntimeDeps pkgs.python3Packages; + postInstall = '' + mkdir -p "$out/${pkgs.python3.sitePackages}/blocksci/Blockchain-Known-Pools" + cp ${knownPools}/pools.json \ + "$out/${pkgs.python3.sitePackages}/blocksci/Blockchain-Known-Pools/pools.json" + ''; + pythonImportsCheck = [ "blocksci" ]; doCheck = false; From ff687037f9f2f452d4673f0a8551d67b12124a88 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:18:01 -0300 Subject: [PATCH 11/25] feat(nix): provide nix-only dev shell and btc check --- flake.nix | 102 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 68 insertions(+), 34 deletions(-) diff --git a/flake.nix b/flake.nix index 2a4cfc91..7b583bfe 100644 --- a/flake.nix +++ b/flake.nix @@ -32,12 +32,57 @@ pycryptodome requests ]; + pytestRegtest = pkgs.python3Packages.buildPythonPackage rec { + pname = "pytest-regtest"; + version = "2.5.1"; + pyproject = true; + + src = pkgs.fetchPypi { + pname = "pytest_regtest"; + inherit version; + hash = "sha256-hPd/GPpwKVN3Zi3pbXIdA7MtBVCcU+E2au7BHb7VbsA="; + }; + + build-system = with pkgs.python3Packages; [ + setuptools + ]; + + dependencies = with pkgs.python3Packages; [ + deepdiff + pytest + ]; + + # nixpkgs 26.05 currently ships deepdiff 8.x while pytest-regtest + # 2.5.1 declares deepdiff >= 9. BlockSci's regression tests exercise + # the plugin successfully with deepdiff 8.x, so relax this until + # nixpkgs updates deepdiff or we add a dedicated deepdiff 9 package. + pythonRelaxDeps = [ "deepdiff" ]; + + pythonImportsCheck = [ "pytest_regtest" ]; + + doCheck = false; + }; + pythonTestDeps = + ps: [ + ps.pytest + ps."pytest-benchmark" + pytestRegtest + ps.pytz + ps.tzlocal + ]; knownPools = pkgs.fetchFromGitHub { owner = "blockchain"; repo = "Blockchain-Known-Pools"; rev = "29ab27c844ebdb63110f8783f73b9decd4abc221"; sha256 = "sha256-nFWprW6+PXDCIW/IjbAm9PNTMs+Clu42gCtfNnTU8QI="; }; + pythonDevEnv = pkgs.python3.withPackages ( + ps: + [ + self.packages.${system}.blockscipy + ] + ++ pythonTestDeps ps + ); mkHeaderOnly = { pname, @@ -141,45 +186,34 @@ }; devShells.default = pkgs.mkShell { - inputsFrom = [ + packages = with pkgs; [ self.packages.${system}.default + pythonDevEnv + clang-tools + cmake + ninja + ruff ]; - nativeBuildInputs = - with pkgs; - [ - python3 - python3Packages.pip - python3Packages.scikit-build-core - clang-tools - ninja - ruff - ] - ++ pythonRuntimeDeps pkgs.python3Packages; + BLOCKSCI_POOLS_JSON = "${knownPools}/pools.json"; + CMAKE_PREFIX_PATH = "${self.packages.${system}.default}"; + }; - buildInputs = with pkgs; [ - python3Packages.pybind11 - howard-hinnant-date + # Initial flake CI intentionally covers BTC regtest only. BCH/LTC + # coverage can be added as follow-up checks once BTC remains stable. + checks.blockscipy-btc = pkgs.runCommand "blocksci-python-btc-tests" { + nativeBuildInputs = [ + self.packages.${system}.default + pythonDevEnv ]; - - shellHook = '' - export LOCAL_PIP="$PWD/.nix-pip" - export PYTHONPATH="$LOCAL_PIP/${pkgs.python3.sitePackages}:$PYTHONPATH" - export PATH="$LOCAL_PIP/bin:$PATH" - - export CMAKE_PREFIX_PATH="$PWD/.nix-install:$CMAKE_PREFIX_PATH" - '' - + ( - if pkgs.stdenv.isDarwin then - '' - export DYLD_LIBRARY_PATH="$PWD/.nix-install/lib:$DYLD_LIBRARY_PATH" - '' - else - '' - export LD_LIBRARY_PATH="$PWD/.nix-install/lib64:$LD_LIBRARY_PATH" - '' - ); - }; + BLOCKSCI_POOLS_JSON = "${knownPools}/pools.json"; + } '' + cp -R ${self}/test "$TMPDIR/test" + chmod -R u+w "$TMPDIR/test" + cd "$TMPDIR/test/blockscipy" + python -m pytest --btc -q + touch "$out" + ''; packages.bitcoin-api-cpp = pkgs.stdenv.mkDerivation { pname = "bitcoin-api-cpp"; From a9b3428f460299094f77566c1bf00bd7f6cc37d3 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:00:38 -0300 Subject: [PATCH 12/25] docs(nix): document nix-only workflow --- nix.md | 70 ++++++++++++++++++++++++------------------- test/requirements.txt | 4 ++- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/nix.md b/nix.md index c55a749c..4d062fad 100644 --- a/nix.md +++ b/nix.md @@ -1,57 +1,67 @@ -# Building BlockSci with Nix +# Building and Testing BlockSci with Nix -This project ships a Nix flake that provides: +This repository is Nix-first. The flake provides: -- Two package derivations (`packages.default` for libblocksci + tools, - `packages.blockscipy` for the Python bindings). -- A development shell (`devShells.default`) that reproduces the upstream - build manual with all native dependencies preinstalled, no system pollution. +- `packages.default`: libblocksci and command-line tools, including `blocksci_parser`. +- `packages.blockscipy`: Python bindings for BlockSci. +- `devShells.default`: a ready-to-use development/test shell with BlockSci, pytest, and benchmark dependencies. The flake is pinned to `nixpkgs-25.11` and targets the four default systems (`x86_64`/`aarch64` × Linux/macOS). -## Build libblocksci and the command-line tools +## Build packages -``` +Build the C++ library and command-line tools: + +```bash nix build ``` -## Build the Python bindings +Build the Python bindings: -``` +```bash nix build .#blockscipy ``` -## Building in the development shell +## Development shell -This workflow builds the project step by step, self-contained under -`.nix-install/`, `.nix-pip/` and `release`. +Enter the shell: -1. Enter the shell: - -``` +```bash nix develop ``` -2. Build and install libblocksci: +Inside the shell, Python imports the Nix-built `blocksci` package directly. No `pip install`, `LOCAL_PIP`, `.nix-pip`, or manual `PYTHONPATH` setup is required. + +Useful smoke checks: ```bash -mkdir release && cd release -cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=../.nix-install .. -make -make install -cd .. +python -c 'import blocksci; print(blocksci.__file__)' +which blocksci_parser ``` -3. Build the Python bindings: +Both should point into `/nix/store`. +## Run Python regression tests + +```bash +nix develop +cd test/blockscipy +python -m pytest --btc -q ``` -pip install --no-build-isolation --prefix=$LOCAL_PIP ./blockscipy + +The tests use synthetic Bitcoin regtest blocks from `test/files/btc/regtest/` and call `blocksci_parser` from the Nix package. +The initial flake check intentionally covers BTC regtest only; BCH/LTC checks can be added later as separate follow-ups. + +## Run benchmarks + +```bash +nix develop +cd test/benchmark +python -m pytest --btc -v --benchmark-autosave --benchmark-warmup=true --benchmark-warmup-iterations=1 --benchmark-sort=name --benchmark-columns=mean,median,max,rounds,iterations ``` -`LOCAL_PIP`, `CMAKE_PREFIX_PATH` and the dynamic-linker search path -(`LD_LIBRARY_PATH`, or `DYLD_LIBRARY_PATH` on macOS) are exported by the -shellHook so the two halves find each other at build and run time. -`--no-build-isolation` makes pip use the `scikit-build-core` backend from the -shell instead of fetching it from PyPI. +## Known-pools data + +`Blockchain-Known-Pools` is fetched by Nix and exposed through `BLOCKSCI_POOLS_JSON`. +The project no longer requires that repository as a Git submodule. diff --git a/test/requirements.txt b/test/requirements.txt index 45c9c2d8..05863878 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,7 @@ +# Test dependencies for non-Nix environments. +# Nix users should use `nix develop`; the flake provides these dependencies. pytest>=3.7.3 pytest-regtest>=1.3.1 tzlocal>=1.5.1 pytz>=2018.5 -pytest-benchmark>=3.2.3 \ No newline at end of file +pytest-benchmark>=3.2.3 From 41242f5bc53b69d4c2ebca15a00a7373c46089d0 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:18:44 -0300 Subject: [PATCH 13/25] chore(repo): ignore local build artifacts --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index a400b07c..18258e32 100644 --- a/.gitignore +++ b/.gitignore @@ -3,13 +3,18 @@ install xcode debug build +bin/ +lib/ .mtime_cache *.DS_Store +*.log Notebooks/blocksci *.so *.dylib *.pyc +__pycache__/ include/blocksci/external +include/blocksci/blocksci_export.h python/blocksci/pybind11 *.egg-info blockscipy/dist/* @@ -17,5 +22,8 @@ blockscipy/dist/* .idea .benchmarks result +result-* .nix-install .nix-pip +LOCAL_PIP/ +test-chain/ From 25c798eead53b9b7d98ce467e45e45a392cc4854 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:44:30 -0300 Subject: [PATCH 14/25] build(blockscipy): disable pybind11 lto extras --- blockscipy/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockscipy/CMakeLists.txt b/blockscipy/CMakeLists.txt index 52363875..7084e41c 100644 --- a/blockscipy/CMakeLists.txt +++ b/blockscipy/CMakeLists.txt @@ -30,7 +30,7 @@ file(GLOB PYTHON_INTERFACE_GENERIC_PROXY_SOURCES "src/generic_proxy/*.cpp") file(GLOB PYTHON_INTERFACE_HEADERS "src/*.hpp") file(GLOB PYTHON_INTERFACE_SOURCES "src/*.cpp") -pybind11_add_module(_blocksci ${PYTHON_INTERFACE_PROXY_HEADERS} ${PYTHON_INTERFACE_PROXY_SOURCES} ${PYTHON_CHAIN_SOURCES} ${PYTHON_CLUSTER_HEADERS} ${PYTHON_CLUSTER_SOURCES} ${PYTHON_CHAIN_HEADERS} ${PYTHON_SCRIPTS_HEADERS} ${PYTHON_SCRIPTS_SOURCES} ${PYTHON_INTERFACE_SOURCES} ${PYTHON_INTERFACE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_SOURCES} ${PYTHON_INTERFACE_GENERIC_PROXY_HEADERS} ${PYTHON_INTERFACE_GENERIC_PROXY_SOURCES}) +pybind11_add_module(_blocksci NO_EXTRAS ${PYTHON_INTERFACE_PROXY_HEADERS} ${PYTHON_INTERFACE_PROXY_SOURCES} ${PYTHON_CHAIN_SOURCES} ${PYTHON_CLUSTER_HEADERS} ${PYTHON_CLUSTER_SOURCES} ${PYTHON_CHAIN_HEADERS} ${PYTHON_SCRIPTS_HEADERS} ${PYTHON_SCRIPTS_SOURCES} ${PYTHON_INTERFACE_SOURCES} ${PYTHON_INTERFACE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_SOURCES} ${PYTHON_INTERFACE_GENERIC_PROXY_HEADERS} ${PYTHON_INTERFACE_GENERIC_PROXY_SOURCES}) set_target_properties(_blocksci PROPERTIES CXX_VISIBILITY_PRESET hidden) set_target_properties(_blocksci PROPERTIES VISIBILITY_INLINES_HIDDEN 1) From 115cc4023e1658c39d584e4239942918be766884 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:52:56 -0300 Subject: [PATCH 15/25] build(blockscipy): reduce gcc binding optimization pressure --- blockscipy/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/blockscipy/CMakeLists.txt b/blockscipy/CMakeLists.txt index 7084e41c..454cb8d7 100644 --- a/blockscipy/CMakeLists.txt +++ b/blockscipy/CMakeLists.txt @@ -39,6 +39,10 @@ add_subdirectory(external) target_compile_options(_blocksci PRIVATE -Wall -Wextra -Wpedantic) +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_compile_options(_blocksci PRIVATE -O1) +endif() + target_link_libraries( _blocksci From dd5acd67180b575dafb3e91332f84516094c50bb Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:00:51 -0300 Subject: [PATCH 16/25] build(blockscipy): respect nix build cores in cmake --- flake.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/flake.nix b/flake.nix index 7b583bfe..56e092b7 100644 --- a/flake.nix +++ b/flake.nix @@ -167,6 +167,17 @@ dontUseCmakeConfigure = true; + preBuild = '' + parallel_level="''${NIX_BUILD_CORES:-1}" + + if [ "$parallel_level" -lt 1 ]; then + parallel_level=1 + fi + + export CMAKE_BUILD_PARALLEL_LEVEL="$parallel_level" + echo "CMAKE_BUILD_PARALLEL_LEVEL=$CMAKE_BUILD_PARALLEL_LEVEL" + ''; + buildInputs = with pkgs; [ howard-hinnant-date self.packages.${system}.default From ac5e8d7e0381c261178325c1cb5ab3f996e737e4 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:01:14 -0300 Subject: [PATCH 17/25] docs(nix): document build parallelism and logs --- nix.md | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/nix.md b/nix.md index 4d062fad..46b9ac7a 100644 --- a/nix.md +++ b/nix.md @@ -14,21 +14,57 @@ The flake is pinned to `nixpkgs-25.11` and targets the four default systems Build the C++ library and command-line tools: ```bash -nix build +nix build -L ``` -Build the Python bindings: +Build the Python bindings on memory-constrained machines: ```bash -nix build .#blockscipy +nix build .#blockscipy -L --cores 1 --max-jobs 1 ``` +Build the Python bindings with more parallelism on machines with more RAM: + +```bash +nix build .#blockscipy -L --cores 4 --max-jobs 1 +``` + +`packages.blockscipy` forwards `NIX_BUILD_CORES` to CMake through +`CMAKE_BUILD_PARALLEL_LEVEL`, so `--cores` controls the internal +CMake/Ninja parallelism used to compile the Python extension. `--max-jobs` +controls how many Nix derivations may build at the same time. Use +`--max-jobs 1` when you want one build running at a time. + +## Run flake checks + +Run the default checks for the current system: + +```bash +nix flake check -L --cores 4 --max-jobs 1 +``` + +On machines with limited RAM, lower the CMake/Ninja parallelism: + +```bash +nix flake check -L --cores 1 --max-jobs 1 +``` + +The current flake check intentionally runs the BTC Python regression suite +only. BCH/LTC checks can be added later as separate follow-ups. + ## Development shell -Enter the shell: +Enter the shell. The shell depends on the Nix-built Python bindings, so the +first invocation may compile `blockscipy` before opening the shell. ```bash -nix develop +nix develop -L --cores 4 --max-jobs 1 +``` + +On memory-constrained machines, use a serial build: + +```bash +nix develop -L --cores 1 --max-jobs 1 ``` Inside the shell, Python imports the Nix-built `blocksci` package directly. No `pip install`, `LOCAL_PIP`, `.nix-pip`, or manual `PYTHONPATH` setup is required. @@ -45,22 +81,57 @@ Both should point into `/nix/store`. ## Run Python regression tests ```bash -nix develop +nix develop -L --cores 4 --max-jobs 1 cd test/blockscipy python -m pytest --btc -q ``` +Or run the same test command without entering an interactive shell: + +```bash +nix develop -L --cores 4 --max-jobs 1 --command sh -c 'cd test/blockscipy && python -m pytest --btc -q -W error::DeprecationWarning' +``` + The tests use synthetic Bitcoin regtest blocks from `test/files/btc/regtest/` and call `blocksci_parser` from the Nix package. -The initial flake check intentionally covers BTC regtest only; BCH/LTC checks can be added later as separate follow-ups. ## Run benchmarks ```bash -nix develop +nix develop -L --cores 4 --max-jobs 1 cd test/benchmark python -m pytest --btc -v --benchmark-autosave --benchmark-warmup=true --benchmark-warmup-iterations=1 --benchmark-sort=name --benchmark-columns=mean,median,max,rounds,iterations ``` +## Logs, cache, and rebuilds + +Nix is quiet on success, especially when outputs are already present in the +store. A command that prints no build output may still have passed. Check the +exit status with: + +```bash +echo $? +``` + +After `nix build`, the `result` symlink points to the built output: + +```bash +readlink result +``` + +Inspect logs from existing build outputs: + +```bash +nix log .#blockscipy +nix log .#checks.$(nix eval --raw --impure --expr builtins.currentSystem).blockscipy-btc +``` + +Force a rebuild when you need fresh live output: + +```bash +nix build .#blockscipy -L --rebuild --cores 4 --max-jobs 1 +nix flake check -L --rebuild --cores 4 --max-jobs 1 +``` + ## Known-pools data `Blockchain-Known-Pools` is fetched by Nix and exposed through `BLOCKSCI_POOLS_JSON`. From 955110bba2f3f4e794493231f9dc119c42060850 Mon Sep 17 00:00:00 2001 From: Edil Medeiros Date: Sat, 27 Jun 2026 19:48:56 -0300 Subject: [PATCH 18/25] bump nixpkgs version --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index f5eb3c02..5ea60b03 100644 --- a/flake.lock +++ b/flake.lock @@ -20,16 +20,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1781509190, - "narHash": "sha256-uJZs9Di8I6ciTp6jiojj0HzlNpBkud8ax5aT/O5aJkw=", + "lastModified": 1782375420, + "narHash": "sha256-wiPYmEuHbJvleW489n6+lamL7JSJg3pcKUYwURU9CkI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d6df3513510aa548c83868fd22bfddd0a8c0a0d4", + "rev": "4062d36ebeae843c750011eef6b61ec9a9dbc9a9", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-25.11", + "ref": "nixos-26.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 56e092b7..ea79b00d 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "BlockSci analysis tool for blockchains"; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; flake-utils.url = "github:numtide/flake-utils"; }; From 0a4efe689069ccad2aa8cf1450fdcf6903491ec4 Mon Sep 17 00:00:00 2001 From: Edil Medeiros Date: Sat, 27 Jun 2026 20:16:13 -0300 Subject: [PATCH 19/25] Add direnv to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 18258e32..abd0cad7 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ result-* .nix-pip LOCAL_PIP/ test-chain/ +/.direnv +/.envrc From e6123f6c03c96c3b3b6f37fa74032ecab1d3ca0e Mon Sep 17 00:00:00 2001 From: Isaque Franklin Date: Sun, 28 Jun 2026 01:23:42 -0300 Subject: [PATCH 20/25] fix: macos native build --- build.md | 21 ++++++++++++++++++--- src/internal/address_index.cpp | 4 ++-- src/internal/hash_index.cpp | 4 ++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/build.md b/build.md index acd88aec..f7dee3dc 100644 --- a/build.md +++ b/build.md @@ -54,10 +54,9 @@ sudo apt install \ ```bash brew install \ - cmake git boost openssl rocksdb google-sparsehash libsecp256k1 \ + cmake git boost openssl rocksdb google-sparsehash secp256k1 \ range-v3 nlohmann-json cereal googletest curl jsoncpp ``` - #### **Arch Linux:** > **Note:** Arch is not supported yet. Its rolling `rocksdb` (≥ 11) removed the @@ -86,6 +85,8 @@ compiled libraries `libjson-rpc-cpp` and `bitcoin-api-cpp`. ## Build libblocksci and the tools +### For general OS + ```bash cmake -B release \ -DCMAKE_BUILD_TYPE=Release \ @@ -94,6 +95,21 @@ cmake -B release \ -DCMAKE_PREFIX_PATH="$PREFIX" cmake --build release cmake --install release + +``` + +### For macOS (Apple silicon) + +```bash +cmake -B release \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX;$(brew --prefix)" \ + -DOPENSSL_ROOT_DIR="$(brew --prefix openssl)" +cmake --build release +cmake --install release ``` `CMAKE_PREFIX_PATH` points CMake at the unpackaged dependencies installed above. @@ -113,7 +129,6 @@ Install into a virtual environment so the build does not touch system Python: python3 -m venv "$PREFIX/venv" source "$PREFIX/venv/bin/activate" ``` - `pip` reads `blockscipy/pyproject.toml`, builds the pybind11 module against the installed C++ library, and pulls the Python runtime dependencies from PyPI: diff --git a/src/internal/address_index.cpp b/src/internal/address_index.cpp index 268d5031..46956036 100644 --- a/src/internal/address_index.cpp +++ b/src/internal/address_index.cpp @@ -71,7 +71,7 @@ namespace blocksci { }); columnDescriptors.emplace_back(rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions()); - rocksdb::DB *dbPtr; + std::unique_ptr dbPtr; std::vector columnHandlePtrs; if (readonly) { rocksdb::Status s = @@ -85,7 +85,7 @@ namespace blocksci { throw std::runtime_error{"Could not open address index with error: " + std::string{s.getState()}}; } } - db = std::unique_ptr(dbPtr); + db = std::move(dbPtr); for (auto handle : columnHandlePtrs) { columnHandles.emplace_back(std::unique_ptr(handle)); } diff --git a/src/internal/hash_index.cpp b/src/internal/hash_index.cpp index b0104d99..c17f5ce9 100644 --- a/src/internal/hash_index.cpp +++ b/src/internal/hash_index.cpp @@ -76,7 +76,7 @@ namespace blocksci { columnDescriptors.emplace_back(rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions{}); columnDescriptors.emplace_back("T", txOptions); - rocksdb::DB *dbPtr; + std::unique_ptr dbPtr; std::vector columnHandlePtrs; if (readonly) { rocksdb::Status s = @@ -90,7 +90,7 @@ namespace blocksci { throw std::runtime_error{"Could not open hash index with error: " + std::string{s.getState()}}; } } - db = std::unique_ptr(dbPtr); + std::move(dbPtr); for (auto handle : columnHandlePtrs) { columnHandles.emplace_back(std::unique_ptr(handle)); } From 03cb590d75865754644676791906fe75c232bcf2 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:45:44 -0300 Subject: [PATCH 21/25] fix(rocksdb): keep db ownership compatible with rocksdb 8 RocksDB 8 opens databases through DB** out parameters. Keep the raw pointer during Open/OpenForReadOnly and transfer ownership into the existing unique_ptr after a successful open. Co-authored-by: Isaque Franklin --- src/internal/address_index.cpp | 4 ++-- src/internal/hash_index.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/internal/address_index.cpp b/src/internal/address_index.cpp index 46956036..268d5031 100644 --- a/src/internal/address_index.cpp +++ b/src/internal/address_index.cpp @@ -71,7 +71,7 @@ namespace blocksci { }); columnDescriptors.emplace_back(rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions()); - std::unique_ptr dbPtr; + rocksdb::DB *dbPtr; std::vector columnHandlePtrs; if (readonly) { rocksdb::Status s = @@ -85,7 +85,7 @@ namespace blocksci { throw std::runtime_error{"Could not open address index with error: " + std::string{s.getState()}}; } } - db = std::move(dbPtr); + db = std::unique_ptr(dbPtr); for (auto handle : columnHandlePtrs) { columnHandles.emplace_back(std::unique_ptr(handle)); } diff --git a/src/internal/hash_index.cpp b/src/internal/hash_index.cpp index c17f5ce9..b0104d99 100644 --- a/src/internal/hash_index.cpp +++ b/src/internal/hash_index.cpp @@ -76,7 +76,7 @@ namespace blocksci { columnDescriptors.emplace_back(rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions{}); columnDescriptors.emplace_back("T", txOptions); - std::unique_ptr dbPtr; + rocksdb::DB *dbPtr; std::vector columnHandlePtrs; if (readonly) { rocksdb::Status s = @@ -90,7 +90,7 @@ namespace blocksci { throw std::runtime_error{"Could not open hash index with error: " + std::string{s.getState()}}; } } - std::move(dbPtr); + db = std::unique_ptr(dbPtr); for (auto handle : columnHandlePtrs) { columnHandles.emplace_back(std::unique_ptr(handle)); } From 16bf63120183d243f8654b2d141382297c0c3c63 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:45:51 -0300 Subject: [PATCH 22/25] fix(build): require cxx17 on core targets Make the C++17 requirement part of the exported target contract so native and Python extension builds inherit the same standard consistently. --- src/CMakeLists.txt | 2 +- src/internal/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cada5e7a..8243668b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -44,7 +44,7 @@ cxx_variadic_templates cxx_template_template_parameters ) -target_compile_features(blocksci PUBLIC ${BLOCKSCI_FEATURE_REQUIREMENTS}) +target_compile_features(blocksci PUBLIC cxx_std_17 ${BLOCKSCI_FEATURE_REQUIREMENTS}) target_compile_options(blocksci PRIVATE -Wall -Wextra -Wpedantic) diff --git a/src/internal/CMakeLists.txt b/src/internal/CMakeLists.txt index a6a9116c..b9a2408e 100644 --- a/src/internal/CMakeLists.txt +++ b/src/internal/CMakeLists.txt @@ -44,7 +44,7 @@ cxx_variadic_templates cxx_template_template_parameters ) -target_compile_features(blocksci_internal PUBLIC ${BLOCKSCI_FEATURE_REQUIREMENTS}) +target_compile_features(blocksci_internal PUBLIC cxx_std_17 ${BLOCKSCI_FEATURE_REQUIREMENTS}) target_compile_options(blocksci_internal PRIVATE -Wall -Wextra -Wpedantic) From 4494bb8d54d298030505ebda71f9f7b28eec1b0e Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:45:59 -0300 Subject: [PATCH 23/25] build(nix): pin rocksdb 8.9.1 Use a pinned RocksDB release compatible with BlockSci's existing C++17 DB** API instead of nixpkgs' newer RocksDB packages. Patch the common RocksDB namespace header with for newer GCC/libstdc++ include strictness. --- flake.nix | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index ea79b00d..acbb7187 100644 --- a/flake.nix +++ b/flake.nix @@ -131,7 +131,6 @@ gtest nlohmann_json openssl - rocksdb secp256k1 sparsehash self.packages.${system}.bitcoin-api-cpp @@ -139,6 +138,7 @@ self.packages.${system}.endian self.packages.${system}.filesystem self.packages.${system}.mio + self.packages.${system}.rocksdb_8_9_1 ]; propagatedBuildInputs = with pkgs; [ @@ -256,6 +256,64 @@ ]; }; + packages.rocksdb_8_9_1 = pkgs.stdenv.mkDerivation { + pname = "rocksdb"; + version = "8.9.1"; + + src = pkgs.fetchFromGitHub { + owner = "facebook"; + repo = "rocksdb"; + rev = "v8.9.1"; + hash = "sha256-Pl7t4FVOvnORWFS+gjy2EEUQlPxjLukWW5I5gzCQwkI="; + }; + + nativeBuildInputs = with pkgs; [ + cmake + ninja + ]; + + propagatedBuildInputs = with pkgs; [ + bzip2 + lz4 + snappy + zlib + zstd + ]; + + postPatch = '' + # GCC 15 no longer accepts relying on transitive cstdint includes. + substituteInPlace include/rocksdb/rocksdb_namespace.h \ + --replace-fail '#pragma once' '#pragma once + + #include ' + ''; + + cmakeFlags = [ + "-DPORTABLE=1" + "-DWITH_JEMALLOC=0" + "-DWITH_JNI=0" + "-DWITH_BENCHMARK_TOOLS=0" + "-DWITH_TESTS=0" + "-DWITH_TOOLS=0" + "-DWITH_CORE_TOOLS=0" + "-DWITH_BZ2=1" + "-DWITH_LZ4=1" + "-DWITH_SNAPPY=1" + "-DWITH_ZLIB=1" + "-DWITH_ZSTD=1" + "-DWITH_GFLAGS=0" + "-DUSE_RTTI=1" + "-DFAIL_ON_WARNINGS=NO" + ]; + + postFixup = '' + if [ -f "$out"/lib/pkgconfig/rocksdb.pc ]; then + substituteInPlace "$out"/lib/pkgconfig/rocksdb.pc \ + --replace-fail '="''${prefix}//' '="/' + fi + ''; + }; + packages.dset = mkHeaderOnly { pname = "dset"; version = "unstable-2015-06-28"; From 2ce8308ad2fefae88f9e7f0a5a064031c1f303f0 Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:46:09 -0300 Subject: [PATCH 24/25] build(deps): support pinned rocksdb for native macos Allow the unpackaged dependency script to build RocksDB 8.9.1 into the local prefix when BLOCKSCI_BUILD_ROCKSDB=1. Document the macOS path so native builds avoid Homebrew's newer RocksDB ABI/API expectations. Co-authored-by: Isaque Franklin --- build.md | 15 +++++++++++-- scripts/install-unpackaged-deps.sh | 36 ++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/build.md b/build.md index f7dee3dc..e2128a77 100644 --- a/build.md +++ b/build.md @@ -54,9 +54,14 @@ sudo apt install \ ```bash brew install \ - cmake git boost openssl rocksdb google-sparsehash secp256k1 \ - range-v3 nlohmann-json cereal googletest curl jsoncpp + cmake git boost openssl google-sparsehash secp256k1 \ + range-v3 nlohmann-json cereal googletest curl jsoncpp \ + bzip2 lz4 snappy zstd ``` + +Homebrew's latest `rocksdb` currently requires newer C++ headers than BlockSci's +supported C++17 build. Build the pinned RocksDB version into `$PREFIX` with the +unpackaged dependency script instead. #### **Arch Linux:** > **Note:** Arch is not supported yet. Its rolling `rocksdb` (≥ 11) removed the @@ -79,6 +84,12 @@ installs them into `$PREFIX`: ./scripts/install-unpackaged-deps.sh "$PREFIX" ``` +On macOS, include the pinned RocksDB build used by CI: + +```bash +BLOCKSCI_BUILD_ROCKSDB=1 ./scripts/install-unpackaged-deps.sh "$PREFIX" +``` + This installs the header-only libraries `mpark/variant`, `wjfilesystem`, `mio`, `dset`, `steinwurf/endian`, `clipp`, and Howard Hinnant's `date`, plus the compiled libraries `libjson-rpc-cpp` and `bitcoin-api-cpp`. diff --git a/scripts/install-unpackaged-deps.sh b/scripts/install-unpackaged-deps.sh index 9d99c2cc..14bd50df 100755 --- a/scripts/install-unpackaged-deps.sh +++ b/scripts/install-unpackaged-deps.sh @@ -83,6 +83,42 @@ if command -v brew >/dev/null 2>&1; then prefix_path="$PREFIX;$(brew --prefix)" fi +if [ "${BLOCKSCI_BUILD_ROCKSDB:-0}" = "1" ]; then + log "facebook/rocksdb 8.9.1" + src="$(fetch facebook/rocksdb v8.9.1)" + sed -i.bak \ + 's|#pragma once|#pragma once\ +\ +#include |' \ + "$src/include/rocksdb/rocksdb_namespace.h" + cmake -S "$src" -B "$src/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$prefix_path" \ + -DPORTABLE=1 \ + -DWITH_JEMALLOC=0 \ + -DWITH_JNI=0 \ + -DWITH_BENCHMARK_TOOLS=0 \ + -DWITH_TESTS=0 \ + -DWITH_TOOLS=0 \ + -DWITH_CORE_TOOLS=0 \ + -DWITH_BZ2=1 \ + -DWITH_LZ4=1 \ + -DWITH_SNAPPY=1 \ + -DWITH_ZLIB=1 \ + -DWITH_ZSTD=1 \ + -DWITH_GFLAGS=0 \ + -DUSE_RTTI=1 \ + -DFAIL_ON_WARNINGS=NO + cmake --build "$src/build" -j "$JOBS" + cmake --install "$src/build" + if [ -f "$PREFIX/lib/pkgconfig/rocksdb.pc" ]; then + sed -i.bak 's|="${prefix}//|="/|g' "$PREFIX/lib/pkgconfig/rocksdb.pc" + fi + prefix_path="$PREFIX;$prefix_path" +fi + log "cinemast/libjson-rpc-cpp 1.4.1 (HTTP client only)" src="$(fetch cinemast/libjson-rpc-cpp v1.4.1)" for pat in 'cmake_minimum_required(VERSION 3.0)' 'cmake_policy(SET CMP0042 OLD)'; do From 79b182c6f0cfcd39d02c5203f5074bcee322796e Mon Sep 17 00:00:00 2001 From: yan <102800044+yan-pi@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:46:17 -0300 Subject: [PATCH 25/25] ci: add github actions build checks Add PR checks for the Nix BTC path plus native Ubuntu and macOS arm64 builds. The native macOS path builds pinned RocksDB and passes explicit CMake package locations to the Python binding build. --- .github/workflows/ci.yml | 148 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..179e0ce1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,148 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + - master + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + nix-build: + name: Nix BTC checks + runs-on: ubuntu-24.04 + timeout-minutes: 120 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v31 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + extra_nix_config: | + experimental-features = nix-command flakes + accept-flake-config = true + + - name: Run flake checks + run: nix flake check -L --cores 1 --max-jobs 1 + + native-linux: + name: Native Linux build + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + PREFIX: ${{ github.workspace }}/install + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential ca-certificates cmake git ninja-build pkg-config \ + libboost-all-dev libssl-dev librocksdb-dev libsparsehash-dev \ + libsecp256k1-dev librange-v3-dev nlohmann-json3-dev libcereal-dev \ + libgtest-dev libcurl4-openssl-dev libjsoncpp-dev \ + python3 python3-dev python3-pip python3-venv + + - name: Build unpackaged dependencies + run: ./scripts/install-unpackaged-deps.sh "$PREFIX" + + - name: Build and install C++ targets + run: | + cmake -S . -B release -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX" + cmake --build release --parallel 2 + cmake --install release + + - name: Build Python bindings + run: | + python3 -m venv "$PREFIX/venv" + "$PREFIX/venv/bin/python" -m pip install --upgrade pip setuptools wheel + CMAKE_PREFIX_PATH="$PREFIX" \ + CMAKE_BUILD_PARALLEL_LEVEL=2 \ + "$PREFIX/venv/bin/python" -m pip install ./blockscipy + + - name: Smoke test native Linux build + run: | + export PATH="$PREFIX/bin:$PREFIX/venv/bin:$PATH" + export LD_LIBRARY_PATH="$PREFIX/lib:${LD_LIBRARY_PATH:-}" + blocksci_parser --help >/dev/null + python -c 'import blocksci; print(blocksci.VERSION)' + + native-macos: + name: Native macOS arm64 build + runs-on: macos-15 + timeout-minutes: 120 + env: + PREFIX: ${{ github.workspace }}/install + BLOCKSCI_BUILD_ROCKSDB: "1" + HOMEBREW_NO_AUTO_UPDATE: "1" + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "1" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + brew install \ + cmake ninja git boost openssl google-sparsehash secp256k1 \ + range-v3 nlohmann-json cereal googletest curl jsoncpp \ + bzip2 lz4 snappy zstd + + - name: Build unpackaged and pinned dependencies + run: ./scripts/install-unpackaged-deps.sh "$PREFIX" + + - name: Build and install C++ targets + run: | + brew_prefix="$(brew --prefix)" + openssl_prefix="$(brew --prefix openssl)" + cmake -S . -B release -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX;$brew_prefix" \ + -DOPENSSL_ROOT_DIR="$openssl_prefix" + cmake --build release --parallel 2 + cmake --install release + + - name: Build Python bindings + run: | + brew_prefix="$(brew --prefix)" + openssl_prefix="$(brew --prefix openssl)" + python3 -m venv "$PREFIX/venv" + "$PREFIX/venv/bin/python" -m pip install --upgrade \ + pip setuptools wheel scikit-build-core pybind11 + pybind11_dir="$("$PREFIX/venv/bin/python" -c 'import pybind11; print(pybind11.get_cmake_dir())')" + test -f "$PREFIX/lib/cmake/blocksci/blocksci-config.cmake" + test -f "$pybind11_dir/pybind11Config.cmake" + CMAKE_BUILD_PARALLEL_LEVEL=2 \ + "$PREFIX/venv/bin/python" -m pip install -v --no-build-isolation ./blockscipy \ + --config-settings=cmake.define.CMAKE_PREFIX_PATH="$PREFIX;$brew_prefix;$pybind11_dir" \ + --config-settings=cmake.define.blocksci_DIR="$PREFIX/lib/cmake/blocksci" \ + --config-settings=cmake.define.pybind11_DIR="$pybind11_dir" \ + --config-settings=cmake.define.OPENSSL_ROOT_DIR="$openssl_prefix" \ + --config-settings=build.verbose=true + + - name: Smoke test native macOS build + run: | + export PATH="$PREFIX/bin:$PREFIX/venv/bin:$PATH" + export DYLD_LIBRARY_PATH="$PREFIX/lib:${DYLD_LIBRARY_PATH:-}" + blocksci_parser --help >/dev/null + python -c 'import blocksci; print(blocksci.VERSION)'