Test/ci workflows

.dockerignore

@@ -0,0 +1,11 @@
+release
+install
+build
+result
+**/.nix-install
+**/.nix-pip
+**/__pycache__
+**/*.egg-info
+**/*.so
+**/*.dylib
+**/dist

.github/workflows/ci.yml

@@ -0,0 +1,148 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - master
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  nix-build:
+    name: Nix BTC checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+            accept-flake-config = true
+
+      - name: Run flake checks
+        run: nix flake check -L --cores 1 --max-jobs 1
+
+  native-linux:
+    name: Native Linux build
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    env:
+      PREFIX: ${{ github.workspace }}/install
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential ca-certificates cmake git ninja-build pkg-config \
+            libboost-all-dev libssl-dev librocksdb-dev libsparsehash-dev \
+            libsecp256k1-dev librange-v3-dev nlohmann-json3-dev libcereal-dev \
+            libgtest-dev libcurl4-openssl-dev libjsoncpp-dev \
+            python3 python3-dev python3-pip python3-venv
+
+      - name: Build unpackaged dependencies
+        run: ./scripts/install-unpackaged-deps.sh "$PREFIX"
+
+      - name: Build and install C++ targets
+        run: |
+          cmake -S . -B release -G Ninja \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+            -DCMAKE_INSTALL_LIBDIR=lib \
+            -DCMAKE_PREFIX_PATH="$PREFIX"
+          cmake --build release --parallel 2
+          cmake --install release
+
+      - name: Build Python bindings
+        run: |
+          python3 -m venv "$PREFIX/venv"
+          "$PREFIX/venv/bin/python" -m pip install --upgrade pip setuptools wheel
+          CMAKE_PREFIX_PATH="$PREFIX" \
+            CMAKE_BUILD_PARALLEL_LEVEL=2 \
+            "$PREFIX/venv/bin/python" -m pip install ./blockscipy
+
+      - name: Smoke test native Linux build
+        run: |
+          export PATH="$PREFIX/bin:$PREFIX/venv/bin:$PATH"
+          export LD_LIBRARY_PATH="$PREFIX/lib:${LD_LIBRARY_PATH:-}"
+          blocksci_parser --help >/dev/null
+          python -c 'import blocksci; print(blocksci.VERSION)'
+
+  native-macos:
+    name: Native macOS arm64 build
+    runs-on: macos-15
+    timeout-minutes: 120
+    env:
+      PREFIX: ${{ github.workspace }}/install
+      BLOCKSCI_BUILD_ROCKSDB: "1"
+      HOMEBREW_NO_AUTO_UPDATE: "1"
+      HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "1"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install system dependencies
+        run: |
+          brew install \
+            cmake ninja git boost openssl google-sparsehash secp256k1 \
+            range-v3 nlohmann-json cereal googletest curl jsoncpp \
+            bzip2 lz4 snappy zstd
+
+      - name: Build unpackaged and pinned dependencies
+        run: ./scripts/install-unpackaged-deps.sh "$PREFIX"
+
+      - name: Build and install C++ targets
+        run: |
+          brew_prefix="$(brew --prefix)"
+          openssl_prefix="$(brew --prefix openssl)"
+          cmake -S . -B release -G Ninja \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+            -DCMAKE_INSTALL_LIBDIR=lib \
+            -DCMAKE_PREFIX_PATH="$PREFIX;$brew_prefix" \
+            -DOPENSSL_ROOT_DIR="$openssl_prefix"
+          cmake --build release --parallel 2
+          cmake --install release
+
+      - name: Build Python bindings
+        run: |
+          brew_prefix="$(brew --prefix)"
+          openssl_prefix="$(brew --prefix openssl)"
+          python3 -m venv "$PREFIX/venv"
+          "$PREFIX/venv/bin/python" -m pip install --upgrade \
+            pip setuptools wheel scikit-build-core pybind11
+          pybind11_dir="$("$PREFIX/venv/bin/python" -c 'import pybind11; print(pybind11.get_cmake_dir())')"
+          test -f "$PREFIX/lib/cmake/blocksci/blocksci-config.cmake"
+          test -f "$pybind11_dir/pybind11Config.cmake"
+          CMAKE_BUILD_PARALLEL_LEVEL=2 \
+            "$PREFIX/venv/bin/python" -m pip install -v --no-build-isolation ./blockscipy \
+              --config-settings=cmake.define.CMAKE_PREFIX_PATH="$PREFIX;$brew_prefix;$pybind11_dir" \
+              --config-settings=cmake.define.blocksci_DIR="$PREFIX/lib/cmake/blocksci" \
+              --config-settings=cmake.define.pybind11_DIR="$pybind11_dir" \
+              --config-settings=cmake.define.OPENSSL_ROOT_DIR="$openssl_prefix" \
+              --config-settings=build.verbose=true
+
+      - name: Smoke test native macOS build
+        run: |
+          export PATH="$PREFIX/bin:$PREFIX/venv/bin:$PATH"
+          export DYLD_LIBRARY_PATH="$PREFIX/lib:${DYLD_LIBRARY_PATH:-}"
+          blocksci_parser --help >/dev/null
+          python -c 'import blocksci; print(blocksci.VERSION)'

.gitignore

@@ -1,20 +1,31 @@
 release
+install
 xcode
 debug
 build
+bin/
+lib/
 .mtime_cache
 *.DS_Store
+*.log
 Notebooks/blocksci
 *.so
 *.dylib
 *.pyc
+__pycache__/
 include/blocksci/external
+include/blocksci/blocksci_export.h
 python/blocksci/pybind11
 *.egg-info
 blockscipy/dist/*
 .pytest_cache
 .idea
 .benchmarks
 result
+result-*
 .nix-install
 .nix-pip
+LOCAL_PIP/
+test-chain/
+/.direnv
+/.envrc

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "Notebooks/blocksci/Blockchain-Known-Pools"]
-	path = blockscipy/blocksci/Blockchain-Known-Pools
-	url = https://github.com/blockchain/Blockchain-Known-Pools
 [submodule "test/testchain-generator"]
 	path = test/testchain-generator
 	url = https://github.com/citp/testchain-generator

blockscipy/CMakeLists.txt

@@ -30,7 +30,7 @@ file(GLOB PYTHON_INTERFACE_GENERIC_PROXY_SOURCES "src/generic_proxy/*.cpp")
 file(GLOB PYTHON_INTERFACE_HEADERS "src/*.hpp")
 file(GLOB PYTHON_INTERFACE_SOURCES "src/*.cpp")
 
-pybind11_add_module(_blocksci ${PYTHON_INTERFACE_PROXY_HEADERS} ${PYTHON_INTERFACE_PROXY_SOURCES} ${PYTHON_CHAIN_SOURCES} ${PYTHON_CLUSTER_HEADERS} ${PYTHON_CLUSTER_SOURCES} ${PYTHON_CHAIN_HEADERS} ${PYTHON_SCRIPTS_HEADERS} ${PYTHON_SCRIPTS_SOURCES} ${PYTHON_INTERFACE_SOURCES} ${PYTHON_INTERFACE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_SOURCES} ${PYTHON_INTERFACE_GENERIC_PROXY_HEADERS} ${PYTHON_INTERFACE_GENERIC_PROXY_SOURCES})
+pybind11_add_module(_blocksci NO_EXTRAS ${PYTHON_INTERFACE_PROXY_HEADERS} ${PYTHON_INTERFACE_PROXY_SOURCES} ${PYTHON_CHAIN_SOURCES} ${PYTHON_CLUSTER_HEADERS} ${PYTHON_CLUSTER_SOURCES} ${PYTHON_CHAIN_HEADERS} ${PYTHON_SCRIPTS_HEADERS} ${PYTHON_SCRIPTS_SOURCES} ${PYTHON_INTERFACE_SOURCES} ${PYTHON_INTERFACE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_HEADERS} ${PYTHON_INTERFACE_SIMPLE_SOURCES} ${PYTHON_INTERFACE_GENERIC_PROXY_HEADERS} ${PYTHON_INTERFACE_GENERIC_PROXY_SOURCES})
 
 set_target_properties(_blocksci PROPERTIES CXX_VISIBILITY_PRESET hidden)
 set_target_properties(_blocksci PROPERTIES VISIBILITY_INLINES_HIDDEN 1)
@@ -39,6 +39,10 @@ add_subdirectory(external)
 
 target_compile_options(_blocksci PRIVATE -Wall -Wextra -Wpedantic)
 
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+  target_compile_options(_blocksci PRIVATE -O1)
+endif()
+
 target_link_libraries(
 	_blocksci
 

blockscipy/blocksci/__init__.py

@@ -19,10 +19,10 @@
 from functools import reduce
 
 import dateparser
+import multiprocess as mp
 import pandas as pd
 import psutil
 from dateutil.relativedelta import relativedelta
-from multiprocess import Pool
 
 from ._blocksci import *
 from ._blocksci import _traverse
@@ -39,6 +39,48 @@
 sys.modules["blocksci.heuristics"] = heuristics
 sys.modules["blocksci.heuristics.change"] = heuristics.change
 
+# pybind11 3 gives enums Python-native ``Enum`` formatting by default
+# (``address_type.pubkeyhash`` / ``<address_type.pubkeyhash: 2>``).
+# BlockSci's public API and regression fixtures historically exposed a
+# human-readable ``str()`` and a compact qualified-name ``repr()``.
+_ADDRESS_TYPE_LABELS = {
+    address_type.pubkey: "Pay to pubkey",
+    address_type.pubkeyhash: "Pay to pubkey hash",
+    address_type.scripthash: "Pay to script hash",
+    address_type.multisig: "Multisig",
+    address_type.multisig_pubkey: "Multisig Public Key",
+    address_type.nonstandard: "Nonstandard",
+    address_type.nulldata: "Null data",
+    address_type.witness_pubkeyhash: "Pay to witness pubkey hash",
+    address_type.witness_scripthash: "Pay to witness script hash",
+    address_type.witness_unknown: "Pay to witness unknown",
+}
+
+_ADDRESS_TYPE_REPR_NAMES = {
+    address_type.pubkey: "pubkey",
+    address_type.pubkeyhash: "pubkeyhash",
+    address_type.scripthash: "scripthash",
+    address_type.multisig: "multisig",
+    address_type.multisig_pubkey: "multisig_pubkey",
+    address_type.nonstandard: "nonstandard",
+    address_type.nulldata: "nulldata",
+    address_type.witness_pubkeyhash: "witness_pubkeyhash",
+    address_type.witness_scripthash: "witness_scripthash",
+    address_type.witness_unknown: "witness_unknown",
+}
+
+
+def _address_type_str(value):
+    return _ADDRESS_TYPE_LABELS.get(value, "Unknown Address Type")
+
+
+def _address_type_repr(value):
+    return f"address_type.{_ADDRESS_TYPE_REPR_NAMES.get(value, 'unknown')}"
+
+
+address_type.__str__ = _address_type_str
+address_type.__repr__ = _address_type_repr
+
 
 class _NoDefault:
     def __repr__(self):
@@ -48,6 +90,15 @@ def __repr__(self):
 MISSING_PARAM = _NoDefault()
 
 
+def _multiprocess_context():
+    # Python 3.12+ warns when forking a multithreaded process. Pytest and
+    # native extension imports can start helper threads, so prefer spawn on
+    # Darwin where fork-without-exec is particularly fragile.
+    if sys.platform == "darwin" and sys.version_info >= (3, 12):
+        return mp.get_context("spawn")
+    return mp.get_context()
+
+
 # Alert the user if the disk space is getting full
 disk_info = os.statvfs("/")
 free_space = (disk_info.f_frsize * disk_info.f_bavail) // (1024**3)
@@ -96,7 +147,7 @@ def real_map_func(input):
         file.seek(0)
         return file
 
-    with Pool(cpu_count - 1) as p:
+    with _multiprocess_context().Pool(cpu_count - 1) as p:
         results_future = p.map_async(real_map_func, segments[1:])
         first = map_func(chain[raw_segments[0][0] : raw_segments[0][1]])
         results = results_future.get()
@@ -865,6 +916,21 @@ class DummyClass:
 loaderDirectory = os.path.dirname(os.path.abspath(inspect.getsourcefile(DummyClass)))
 
 
+def _load_pool_data():
+    import json
+
+    pools_path = os.environ.get(
+        "BLOCKSCI_POOLS_JSON",
+        os.path.join(loaderDirectory, "Blockchain-Known-Pools", "pools.json"),
+    )
+    try:
+        with open(pools_path) as f:
+            return json.load(f)
+    except FileNotFoundError:
+        logging.warning("BlockSci miner labels disabled: pools.json not found at %s", pools_path)
+        return {"payout_addresses": {}, "coinbase_tags": {}}
+
+
 def get_miner(block) -> str:
     """
     Get the miner of the block based on the text in the coinbase transaction
@@ -874,20 +940,18 @@ def get_miner(block) -> str:
     global pool_data
     global coinbase_tag_re
     if first_miner_run:
-        import json
-
-        with open(loaderDirectory + "/Blockchain-Known-Pools/pools.json") as f:
-            pool_data = json.load(f)
-        addresses = [block._access.address_from_string(addr_string) for addr_string in pool_data["payout_addresses"]]
-        tagged_addresses = {
-            pointer: pool_data["payout_addresses"][address]
-            for address in addresses
-            if address in pool_data["payout_addresses"]
-        }
-        coinbase_tag_re = re.compile("|".join(map(re.escape, pool_data["coinbase_tags"])))
+        pool_data = _load_pool_data()
+        tagged_addresses = {}
+        for addr_string, pool_info in pool_data.get("payout_addresses", {}).items():
+            try:
+                tagged_addresses[block._access.address_from_string(addr_string)] = pool_info
+            except Exception:
+                logging.debug("Skipping invalid miner payout address %s", addr_string, exc_info=True)
+        coinbase_tags = pool_data.get("coinbase_tags", {})
+        coinbase_tag_re = re.compile("|".join(map(re.escape, coinbase_tags))) if coinbase_tags else None
         first_miner_run = False
     coinbase = block.coinbase_param.decode("utf_8", "replace")
-    tag_matches = re.findall(coinbase_tag_re, coinbase)
+    tag_matches = re.findall(coinbase_tag_re, coinbase) if coinbase_tag_re is not None else []
     if tag_matches:
         return pool_data["coinbase_tags"][tag_matches[0]]["name"]
     for txout in block.coinbase_tx.outs:

blockscipy/src/blocksci_range_type.cpp

@@ -29,7 +29,7 @@ struct BlocksciRangeTypeObjectCaster {
 
 struct BlockSciRangeToIterator {
   template <typename T> BlocksciIteratorType operator()(T &&t) {
-    return BlocksciIteratorType{RawIterator<ranges::range_value_type_t<T>>{std::forward<T>(t)}};
+    return BlocksciIteratorType{RawIterator<ranges::range_value_t<T>>{std::forward<T>(t)}};
   }
 };
 

blockscipy/src/blocksci_type_converter.cpp

@@ -20,18 +20,18 @@
 
 #include <range/v3/view/any_view.hpp>
 
-blocksci::AnyScript BlockSciTypeConverter::operator()(const blocksci::Address &address) {
+blocksci::AnyScript BlockSciTypeConverter::operator()(const blocksci::Address &address) const {
   return address.getScript();
 }
 
-RawRange<blocksci::Input> BlockSciTypeConverter::operator()(const blocksci::InputRange &val) {
+RawRange<blocksci::Input> BlockSciTypeConverter::operator()(const blocksci::InputRange &val) const {
   return ranges::any_view<blocksci::Input, random_access_sized>{val};
 }
 
-RawRange<blocksci::Output> BlockSciTypeConverter::operator()(const blocksci::OutputRange &val) {
+RawRange<blocksci::Output> BlockSciTypeConverter::operator()(const blocksci::OutputRange &val) const {
   return ranges::any_view<blocksci::Output, random_access_sized>{val};
 }
 
-RawRange<blocksci::Block> BlockSciTypeConverter::operator()(const blocksci::BlockRange &val) {
+RawRange<blocksci::Block> BlockSciTypeConverter::operator()(const blocksci::BlockRange &val) const {
   return ranges::any_view<blocksci::Block, random_access_sized>{val};
-}
+}