From 84bea3bee7c2351b38b6340bab2bfc7fe68ed831 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Tue, 11 Nov 2025 18:30:12 -0800 Subject: [PATCH 01/16] wip --- python/pyproject.toml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index a490f6699..2ac82cbfa 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + [project] name = "sift_stack_py" version = "0.9.1" @@ -44,6 +48,9 @@ Repository = "https://github.com/sift-stack/sift/tree/main/python" Changelog = "https://github.com/sift-stack/sift/tree/main/python/CHANGELOG.md" [project.optional-dependencies] + + +[tool.sift.extras] # development development = [ "grpcio-testing~=1.13", @@ -55,6 +62,7 @@ development = [ "pytest-mock==3.14.0", "pytest-dotenv==0.5.2", "ruff~=0.12.10", + "tomlkit~=0.13.3" ] build = ["pdoc==14.5.0", "build==1.2.1"] docs = ["mkdocs", @@ -75,9 +83,17 @@ hdf5 = ["h5py~=3.11", "polars~=1.8"] data_review = ["pyarrow>=17.0.0"] # Ensure any new user build extras are added to .github/workflows/python_build.yaml -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +# Combine and alias extras +[tool.sift.extras.combine] +# aliases +dev = ["development"] +sift-stream-bindings = ["sift-stream"] + +# combinations +file-imports = ["tdms", "rosbags", "hdf5"] +all = ["openssl", "sift-stream", "file-imports"] + +all-dev = ["development", "build", "docs", "all"] [tool.mypy] python_version = "3.10" # Use the Python 3.10 type checker since we are using eval-type-backport and `from __future__ import annotations` From 51693669b999557afb9e1b6a736fa9a56293299a Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Tue, 11 Nov 2025 18:38:10 -0800 Subject: [PATCH 02/16] add autogen --- python/pyproject.toml | 106 +++++++++++++++++++++++++++++- python/scripts/generate_extras.py | 62 +++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 python/scripts/generate_extras.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 2ac82cbfa..8da2a94e9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -48,7 +48,111 @@ Repository = "https://github.com/sift-stack/sift/tree/main/python" Changelog = "https://github.com/sift-stack/sift/tree/main/python/CHANGELOG.md" [project.optional-dependencies] - +all = [ + 'cffi~=1.14', + 'h5py~=3.11', + 'npTDMS~=1.9', + 'polars~=1.8', + 'pyOpenSSL<24.0.0', + 'rosbags~=0.0', + 'sift-stream-bindings>=0.2.0-rc', + 'types-pyOpenSSL<24.0.0', +] +all-dev = [ + 'build==1.2.1', + 'cffi~=1.14', + 'griffe-pydantic', + 'grpcio-testing~=1.13', + 'h5py~=3.11', + 'mike', + 'mkdocs', + 'mkdocs-api-autonav', + 'mkdocs-include-markdown-plugin', + 'mkdocs-jupyter', + 'mkdocs-material', + 'mkdocstrings[python]', + 'mypy==1.10.0', + 'npTDMS~=1.9', + 'pdoc==14.5.0', + 'polars~=1.8', + 'pyOpenSSL<24.0.0', + 'pyright==1.1.386', + 'pytest-asyncio==0.23.7', + 'pytest-benchmark==4.0.0', + 'pytest-dotenv==0.5.2', + 'pytest-mock==3.14.0', + 'pytest==8.2.2', + 'rosbags~=0.0', + 'ruff~=0.12.10', + 'sift-stream-bindings>=0.2.0-rc', + 'tomlkit~=0.13.3', + 'types-pyOpenSSL<24.0.0', +] +build = [ + 'build==1.2.1', + 'pdoc==14.5.0', +] +dev = [ + 'grpcio-testing~=1.13', + 'mypy==1.10.0', + 'pyright==1.1.386', + 'pytest-asyncio==0.23.7', + 'pytest-benchmark==4.0.0', + 'pytest-dotenv==0.5.2', + 'pytest-mock==3.14.0', + 'pytest==8.2.2', + 'ruff~=0.12.10', + 'tomlkit~=0.13.3', +] +development = [ + 'grpcio-testing~=1.13', + 'mypy==1.10.0', + 'pyright==1.1.386', + 'pytest-asyncio==0.23.7', + 'pytest-benchmark==4.0.0', + 'pytest-dotenv==0.5.2', + 'pytest-mock==3.14.0', + 'pytest==8.2.2', + 'ruff~=0.12.10', + 'tomlkit~=0.13.3', +] +docs = [ + 'griffe-pydantic', + 'mike', + 'mkdocs', + 'mkdocs-api-autonav', + 'mkdocs-include-markdown-plugin', + 'mkdocs-jupyter', + 'mkdocs-material', + 'mkdocstrings[python]', +] +file-imports = [ + 'h5py~=3.11', + 'npTDMS~=1.9', + 'polars~=1.8', + 'rosbags~=0.0', +] +hdf5 = [ + 'h5py~=3.11', + 'polars~=1.8', +] +openssl = [ + 'cffi~=1.14', + 'pyOpenSSL<24.0.0', + 'types-pyOpenSSL<24.0.0', +] +rosbags = [ + 'rosbags~=0.0', +] +sift-stream = [ + 'sift-stream-bindings>=0.2.0-rc', +] +sift-stream-bindings = [ + 'sift-stream-bindings>=0.2.0-rc', +] +tdms = [ + 'npTDMS~=1.9', +] [tool.sift.extras] # development diff --git a/python/scripts/generate_extras.py b/python/scripts/generate_extras.py new file mode 100644 index 000000000..84f09ea04 --- /dev/null +++ b/python/scripts/generate_extras.py @@ -0,0 +1,62 @@ +import sys +from pathlib import Path + +import tomlkit + +pyproject = Path("../pyproject.toml") +if not pyproject.exists(): + sys.exit(f"❌ No pyproject.toml found at {pyproject.resolve()}") + +# Parse preserving comments and formatting +doc = tomlkit.parse(pyproject.read_text()) + +try: + tool_sift = doc["tool"]["sift"]["extras"] +except KeyError: + sys.exit("❌ No [tool.sift.extras] section found in pyproject.toml") + +# Split atomic and combined definitions +combine_section = tool_sift.get("combine", {}) +atomic_extras = {k: v for k, v in tool_sift.items() if k != "combine"} + +# Recursive resolver for nested combines +def resolve(name, stack=None): + """Recursively resolve nested combine groups.""" + if stack is None: + stack = [] + if name in stack: + raise ValueError(f"Cyclic combine detected: {' -> '.join(stack + [name])}") + if name in atomic_extras: + return list(atomic_extras[name]) + if name in combine_section: + deps = [] + for sub in combine_section[name]: + deps.extend(resolve(sub, stack + [name])) + return deps + raise KeyError(f"Unknown group '{name}' referenced in combine") + +# Build final extras dictionary +final_extras = {} +for name in list(atomic_extras) + list(combine_section): + deps = resolve(name) + final_extras[name] = sorted(set(deps)) + +# Inject into [project.optional-dependencies] +project = doc.setdefault("project", tomlkit.table()) +project["optional-dependencies"] = tomlkit.table() + + +# Write arrays in sorted order with clean formatting +for name in sorted(final_extras): + deps = final_extras[name] + arr = tomlkit.array(deps) + arr.multiline(True) # each dependency on its own line + # arr.indent(2) # 4-space indentation + arr.as_string() + # arr.trailing_comma(True) # trailing commas for cleaner diffs + project["optional-dependencies"][name] = arr + +# Dump back to file +pyproject.write_text(tomlkit.dumps(doc)) + +print("✅ Updated [project.optional-dependencies] with formatted, nested extras") From ea5caf8a372cb475088dc0f84fa5885bab72936d Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Tue, 11 Nov 2025 18:44:00 -0800 Subject: [PATCH 03/16] clean up --- python/pyproject.toml | 6 +++++- python/scripts/generate_extras.py | 21 ++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 8da2a94e9..1e6a5d125 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -48,6 +48,7 @@ Repository = "https://github.com/sift-stack/sift/tree/main/python" Changelog = "https://github.com/sift-stack/sift/tree/main/python/CHANGELOG.md" [project.optional-dependencies] +# AUTO GENERATED EXTRAS — EDIT [tool.sift.extras] ONLY all = [ 'cffi~=1.14', 'h5py~=3.11', @@ -155,6 +156,8 @@ tdms = [ ] [tool.sift.extras] +# Extras configurations defined here will be available in [project.optional-dependencies] + # development development = [ "grpcio-testing~=1.13", @@ -187,8 +190,9 @@ hdf5 = ["h5py~=3.11", "polars~=1.8"] data_review = ["pyarrow>=17.0.0"] # Ensure any new user build extras are added to .github/workflows/python_build.yaml -# Combine and alias extras [tool.sift.extras.combine] +# Combined extras configurations and aliases can be defined here + # aliases dev = ["development"] sift-stream-bindings = ["sift-stream"] diff --git a/python/scripts/generate_extras.py b/python/scripts/generate_extras.py index 84f09ea04..4ff8989d9 100644 --- a/python/scripts/generate_extras.py +++ b/python/scripts/generate_extras.py @@ -43,20 +43,27 @@ def resolve(name, stack=None): # Inject into [project.optional-dependencies] project = doc.setdefault("project", tomlkit.table()) -project["optional-dependencies"] = tomlkit.table() +# Create the optional-dependencies table +opt_table = tomlkit.table() -# Write arrays in sorted order with clean formatting +# Add a header comment BEFORE the section +opt_table.trivia.indent = "" +opt_table.trivia.comment = "\n# AUTO GENERATED EXTRAS — EDIT [tool.sift.extras] ONLY" + +# Write arrays in sorted order for name in sorted(final_extras): deps = final_extras[name] arr = tomlkit.array(deps) - arr.multiline(True) # each dependency on its own line - # arr.indent(2) # 4-space indentation + arr.multiline(True) arr.as_string() - # arr.trailing_comma(True) # trailing commas for cleaner diffs - project["optional-dependencies"][name] = arr + opt_table[name] = arr + + +# Assign back to project +project["optional-dependencies"] = opt_table # Dump back to file pyproject.write_text(tomlkit.dumps(doc)) -print("✅ Updated [project.optional-dependencies] with formatted, nested extras") +print("✅ Updated [project.optional-dependencies]s") From 35d25d33e2647a135b3db99768b41165d8f9e845 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 14:20:00 -0800 Subject: [PATCH 04/16] add hook and update extras references --- .githooks/pre-push | 3 +++ .githooks/pre-push-python/extras.sh | 36 +++++++++++++++++++++++++ .github/workflows/python_build_docs.yml | 8 +++--- .github/workflows/python_ci.yaml | 2 +- .github/workflows/python_release.yaml | 10 +++---- python/pyproject.toml | 8 +++--- python/scripts/build_utils.py | 4 ++- python/scripts/dev | 13 ++++++++- python/scripts/generate_extras.py | 2 ++ 9 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 .githooks/pre-push-python/extras.sh diff --git a/.githooks/pre-push b/.githooks/pre-push index ba7f5ac86..f8bec06c3 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -13,6 +13,9 @@ if [[ -n "$python_changed_files" ]]; then echo "Running Python stub checks..." bash "$GITHOOKS_DIR/pre-push-python/stubs.sh" + + echo "Running Python extras checks..." + bash "$GITHOOKS_DIR/pre-push-python/extras.sh" fi # Check for changes in Rust bindings files diff --git a/.githooks/pre-push-python/extras.sh b/.githooks/pre-push-python/extras.sh new file mode 100644 index 000000000..6987f9ee4 --- /dev/null +++ b/.githooks/pre-push-python/extras.sh @@ -0,0 +1,36 @@ +# ensure generated pyproject.toml extras are up-to-date + +# Store the root directory of the repository +REPO_ROOT="$(git rev-parse --show-toplevel)" +PYTHON_DIR="$REPO_ROOT/python" +PYPROJECT_FILE="$PYTHON_DIR/pyproject.toml" + +# Function to check if pyproject.toml has changed +check_extras_changes() { + local target_path="$1" + local changed_files=$(git status --porcelain "$target_path" || true) + + if [ -n "$changed_files" ]; then + echo "ERROR: Generated pyproject.toml extras are not up-to-date. Please commit the changed files:" + echo "$changed_files" + exit 1 + fi +} + +# Function to generate Python extras +generate_python_extras() { + echo "Generating Python extras..." + cd "$PYTHON_DIR" + + if [[ ! -d "$PYTHON_DIR/venv" ]]; then + echo "Running bootstrap script..." + bash ./scripts/dev bootstrap + fi + + bash ./scripts/dev gen-extras + check_extras_changes "$PYPROJECT_FILE" +} + +generate_python_extras + +echo "All extras are up-to-date." diff --git a/.github/workflows/python_build_docs.yml b/.github/workflows/python_build_docs.yml index 912cb3d33..a654ea3d7 100644 --- a/.github/workflows/python_build_docs.yml +++ b/.github/workflows/python_build_docs.yml @@ -2,7 +2,7 @@ name: Build and Deploy Python Docs (Dev) on: pull_request: - types: [opened, synchronize, closed] + types: [ opened, synchronize, closed ] paths: - 'python/docs/**' - 'python/lib/**' @@ -36,7 +36,7 @@ jobs: - name: Install dependencies run: | cd python - pip install -e .[docs,development] + pip install -e .[docs-build] - name: Extract version id: version @@ -91,8 +91,8 @@ jobs: - name: Install dependencies run: | cd python - pip install -e .[docs,development] - + pip install -e .[docs-build] + - name: Fetch gh-pages branch run: git fetch origin gh-pages --depth=1 diff --git a/.github/workflows/python_ci.yaml b/.github/workflows/python_ci.yaml index 2bb3579c7..cde4ad58b 100644 --- a/.github/workflows/python_ci.yaml +++ b/.github/workflows/python_ci.yaml @@ -30,7 +30,7 @@ jobs: id: install run: | python -m pip install --upgrade pip - pip install '.[development,data_review,openssl,tdms,rosbags,hdf5,sift-stream]' + pip install '.[dev-all]' - name: Lint run: | diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml index fad61f3d7..e37f65f80 100644 --- a/.github/workflows/python_release.yaml +++ b/.github/workflows/python_release.yaml @@ -17,13 +17,13 @@ jobs: publish-to-pypi: name: Upload release to PyPI - needs: [python-ci, build-offline-archives] + needs: [ python-ci, build-offline-archives ] runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/p/sift_py permissions: - id-token: write + id-token: write steps: - name: Download distributions uses: actions/download-artifact@v4 @@ -38,7 +38,7 @@ jobs: create-github-release: name: Create GitHub Release - needs: [build-offline-archives] + needs: [ build-offline-archives ] runs-on: ubuntu-latest permissions: contents: write @@ -110,7 +110,7 @@ jobs: - name: Install dependencies run: | cd python - pip install -e .[docs,development] + pip install -e .[docs-build] - name: Extract version and check if stable id: version @@ -125,7 +125,7 @@ jobs: MINOR=${BASH_REMATCH[2]} SUFFIX=${BASH_REMATCH[3]} VERSION="v${MAJOR}.${MINOR}" - + # Check if this is a stable release (no suffix like -alpha, -beta, -rc) if [[ -z "$SUFFIX" ]]; then # Stable release - use 'latest' alias and make visible diff --git a/python/pyproject.toml b/python/pyproject.toml index 1e6a5d125..7d62de587 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -187,8 +187,7 @@ tdms = ["npTDMS~=1.9"] rosbags = ["rosbags~=0.0"] sift-stream = ["sift-stream-bindings>=0.2.0-rc"] hdf5 = ["h5py~=3.11", "polars~=1.8"] -data_review = ["pyarrow>=17.0.0"] -# Ensure any new user build extras are added to .github/workflows/python_build.yaml +data-review = ["pyarrow>=17.0.0"] [tool.sift.extras.combine] # Combined extras configurations and aliases can be defined here @@ -199,9 +198,10 @@ sift-stream-bindings = ["sift-stream"] # combinations file-imports = ["tdms", "rosbags", "hdf5"] -all = ["openssl", "sift-stream", "file-imports"] +all = ["openssl", "sift-stream", "file-imports", "data-review"] -all-dev = ["development", "build", "docs", "all"] +dev-all = ["development", "build", "docs", "all"] +docs-build = ["development", "docs"] [tool.mypy] python_version = "3.10" # Use the Python 3.10 type checker since we are using eval-type-backport and `from __future__ import annotations` diff --git a/python/scripts/build_utils.py b/python/scripts/build_utils.py index 43225ac8e..cfa085a25 100644 --- a/python/scripts/build_utils.py +++ b/python/scripts/build_utils.py @@ -104,7 +104,9 @@ def main(): # Get all extras from the wheel extras = get_extras_from_wheel(str(wheel_file)) - combinations = get_extra_combinations(extras, exclude=["development", "docs"]) + combinations = get_extra_combinations( + extras, exclude=["development", "docs", "dev", "dev-all", "docs-build"] + ) # Test base installation first test_install( diff --git a/python/scripts/dev b/python/scripts/dev index 7a95a5ced..6a610805b 100755 --- a/python/scripts/dev +++ b/python/scripts/dev @@ -20,6 +20,7 @@ Subcommands: pyright Runs 'pyright lib' for type checking check Runs all python linting checks gen-stubs Generates pyi stubs for sift_client synchronous wrappers + gen-extras Generates optional-dependencies in pyproject.toml mypy-stubs Runs stubtest (mypy) on the generated pyi stubs pip-install Install project dependencies test Execute tests @@ -34,7 +35,7 @@ EOT pip_install() { source venv/bin/activate - pip install '.[development,build,docs,openssl,tdms,rosbags,sift-stream,hdf5]' + pip install '.[all-dev]' pip install -e . } @@ -119,6 +120,13 @@ mypy_stubs() { stubtest sift_client.resources.sync_stubs } +gen_extras() { + source venv/bin/activate + cd scripts + python3 generate_extras.py + cd .. +} + shift "$((OPTIND - 1))" case "$1" in @@ -168,6 +176,9 @@ case "$1" in gen-stubs) gen_stubs ;; + gen-extras) + gen_extras + ;; mypy-stubs) mypy_stubs ;; diff --git a/python/scripts/generate_extras.py b/python/scripts/generate_extras.py index 4ff8989d9..1aeca748a 100644 --- a/python/scripts/generate_extras.py +++ b/python/scripts/generate_extras.py @@ -19,6 +19,7 @@ combine_section = tool_sift.get("combine", {}) atomic_extras = {k: v for k, v in tool_sift.items() if k != "combine"} + # Recursive resolver for nested combines def resolve(name, stack=None): """Recursively resolve nested combine groups.""" @@ -35,6 +36,7 @@ def resolve(name, stack=None): return deps raise KeyError(f"Unknown group '{name}' referenced in combine") + # Build final extras dictionary final_extras = {} for name in list(atomic_extras) + list(combine_section): From 84a91cd575e24ffefa998a72a60502afaa5ca8da Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 14:55:17 -0800 Subject: [PATCH 05/16] update hooks --- .githooks/pre-push | 2 +- .githooks/pre-push-python/extras.sh | 2 +- .githooks/pre-push-python/fmt-lint.sh | 2 +- .githooks/pre-push-python/stubs.sh | 2 +- python/pyproject.toml | 34 +++++++++++++-------------- python/scripts/generate_extras.py | 2 +- 6 files changed, 22 insertions(+), 22 deletions(-) mode change 100644 => 100755 .githooks/pre-push-python/extras.sh diff --git a/.githooks/pre-push b/.githooks/pre-push index f8bec06c3..0c78ed867 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -5,7 +5,7 @@ REPO_ROOT="$(git rev-parse --show-toplevel)" GITHOOKS_DIR="$REPO_ROOT/.githooks" # Check for changes in Python files -python_changed_files=($(git diff --name-only --diff-filter=ACM | grep '^python/lib/sift_client/' || true)) +python_changed_files=($(git diff --name-only --diff-filter=ACM | grep '^python/' || true)) if [[ -n "$python_changed_files" ]]; then echo "Python files changed, running Python formatting and linting..." diff --git a/.githooks/pre-push-python/extras.sh b/.githooks/pre-push-python/extras.sh old mode 100644 new mode 100755 index 6987f9ee4..362e3a42a --- a/.githooks/pre-push-python/extras.sh +++ b/.githooks/pre-push-python/extras.sh @@ -11,7 +11,7 @@ check_extras_changes() { local changed_files=$(git status --porcelain "$target_path" || true) if [ -n "$changed_files" ]; then - echo "ERROR: Generated pyproject.toml extras are not up-to-date. Please commit the changed files:" + echo "❌ ERROR: Generated pyproject.toml extras are not up-to-date. Please commit the changed files:" echo "$changed_files" exit 1 fi diff --git a/.githooks/pre-push-python/fmt-lint.sh b/.githooks/pre-push-python/fmt-lint.sh index 575a4475b..5ff8994df 100644 --- a/.githooks/pre-push-python/fmt-lint.sh +++ b/.githooks/pre-push-python/fmt-lint.sh @@ -25,7 +25,7 @@ changed_files=$(git status --porcelain python/lib/sift_client/ | grep -E '\.py$' if [ -n "$changed_files" ]; then echo "" - echo "ERROR: Formatting/linting made changes to the following files:" + echo "❌ ERROR: Formatting/linting made changes to the following files:" echo "$changed_files" echo "" echo "Please commit these changes before pushing." diff --git a/.githooks/pre-push-python/stubs.sh b/.githooks/pre-push-python/stubs.sh index 42d6d712b..6efdab8f6 100644 --- a/.githooks/pre-push-python/stubs.sh +++ b/.githooks/pre-push-python/stubs.sh @@ -11,7 +11,7 @@ check_stub_changes() { local changed_files=$(git status --porcelain "$target_path" | grep -E '\.pyi$' || true) if [ -n "$changed_files" ]; then - echo "ERROR: Generated python stubs are not up-to-date. Please commit the changed files:" + echo "❌ ERROR: Generated python stubs are not up-to-date. Please commit the changed files:" echo "$changed_files" exit 1 fi diff --git a/python/pyproject.toml b/python/pyproject.toml index 7d62de587..38b9b1523 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -59,7 +59,23 @@ all = [ 'sift-stream-bindings>=0.2.0-rc', 'types-pyOpenSSL<24.0.0', ] -all-dev = [ +build = [ + 'build==1.2.1', + 'pdoc==14.5.0', +] +dev = [ + 'grpcio-testing~=1.13', + 'mypy==1.10.0', + 'pyright==1.1.386', + 'pytest-asyncio==0.23.7', + 'pytest-benchmark==4.0.0', + 'pytest-dotenv==0.5.2', + 'pytest-mock==3.14.0', + 'pytest==8.2.2', + 'ruff~=0.12.10', + 'tomlkit~=0.13.3', +] +dev-all = [ 'build==1.2.1', 'cffi~=1.14', 'griffe-pydantic', @@ -89,22 +105,6 @@ all-dev = [ 'tomlkit~=0.13.3', 'types-pyOpenSSL<24.0.0', ] -build = [ - 'build==1.2.1', - 'pdoc==14.5.0', -] -dev = [ - 'grpcio-testing~=1.13', - 'mypy==1.10.0', - 'pyright==1.1.386', - 'pytest-asyncio==0.23.7', - 'pytest-benchmark==4.0.0', - 'pytest-dotenv==0.5.2', - 'pytest-mock==3.14.0', - 'pytest==8.2.2', - 'ruff~=0.12.10', - 'tomlkit~=0.13.3', -] development = [ 'grpcio-testing~=1.13', 'mypy==1.10.0', diff --git a/python/scripts/generate_extras.py b/python/scripts/generate_extras.py index 1aeca748a..3e4ec0272 100644 --- a/python/scripts/generate_extras.py +++ b/python/scripts/generate_extras.py @@ -68,4 +68,4 @@ def resolve(name, stack=None): # Dump back to file pyproject.write_text(tomlkit.dumps(doc)) -print("✅ Updated [project.optional-dependencies]s") +print("Updated [project.optional-dependencies]") From 35a77937db137f53257ece03db95707be20eb3ef Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 14:59:51 -0800 Subject: [PATCH 06/16] test hooks --- .githooks/pre-push | 9 +++++++++ python/pyproject.toml | 3 +++ 2 files changed, 12 insertions(+) diff --git a/.githooks/pre-push b/.githooks/pre-push index 0c78ed867..c5b3c3157 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -7,6 +7,10 @@ GITHOOKS_DIR="$REPO_ROOT/.githooks" # Check for changes in Python files python_changed_files=($(git diff --name-only --diff-filter=ACM | grep '^python/' || true)) +echo "Changed Python files:" +echo " ${python_changed_files[*]}" +echo + if [[ -n "$python_changed_files" ]]; then echo "Python files changed, running Python formatting and linting..." bash "$GITHOOKS_DIR/pre-push-python/fmt-lint.sh" @@ -21,6 +25,11 @@ fi # Check for changes in Rust bindings files bindings_changed_files=($(git diff --name-only --diff-filter=ACM | grep '^rust/crates/sift_stream_bindings/src/' || true)) +echo +echo "Changed SiftStream Python bindings files:" +echo " ${bindings_changed_files[*]}" +echo + if [[ -n "$bindings_changed_files" ]]; then echo "Rust bindings files changed, running Rust stub checks..." bash "$GITHOOKS_DIR/pre-push-rust/stubs.sh" diff --git a/python/pyproject.toml b/python/pyproject.toml index 38b9b1523..2e18b8199 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -117,6 +117,9 @@ development = [ 'ruff~=0.12.10', 'tomlkit~=0.13.3', ] + + + docs = [ 'griffe-pydantic', 'mike', From ffddd03303b65141a1af7d4da415b7d7b9524217 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 15:04:18 -0800 Subject: [PATCH 07/16] test hooks --- .githooks/pre-push | 45 +++++++++++++++++---------- .githooks/pre-push-python/extras.sh | 11 ++++--- .githooks/pre-push-python/fmt-lint.sh | 14 ++++----- .githooks/pre-push-python/stubs.sh | 11 ++++--- .githooks/pre-push-rust/stubs.sh | 9 +++--- python/pyproject.toml | 2 -- 6 files changed, 52 insertions(+), 40 deletions(-) diff --git a/.githooks/pre-push b/.githooks/pre-push index c5b3c3157..0313d0275 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -4,36 +4,49 @@ REPO_ROOT="$(git rev-parse --show-toplevel)" GITHOOKS_DIR="$REPO_ROOT/.githooks" -# Check for changes in Python files -python_changed_files=($(git diff --name-only --diff-filter=ACM | grep '^python/' || true)) - -echo "Changed Python files:" -echo " ${python_changed_files[*]}" +echo "========================================" +echo " Pre-Push Checks" +echo "========================================" echo +# Check for changes in Python files +# Compare against the remote branch being pushed to +python_changed_files=($(git diff --name-only --diff-filter=ACM @{upstream}... | grep '^python/' || true)) + +echo "→ Python files changed:" if [[ -n "$python_changed_files" ]]; then - echo "Python files changed, running Python formatting and linting..." + echo " ${python_changed_files[*]}" + echo + echo " [1/3] Formatting and linting..." bash "$GITHOOKS_DIR/pre-push-python/fmt-lint.sh" - echo "Running Python stub checks..." + echo " [2/3] Stub generation..." bash "$GITHOOKS_DIR/pre-push-python/stubs.sh" - echo "Running Python extras checks..." + echo " [3/3] Extras validation..." bash "$GITHOOKS_DIR/pre-push-python/extras.sh" + echo +else + echo " (none)" + echo fi # Check for changes in Rust bindings files -bindings_changed_files=($(git diff --name-only --diff-filter=ACM | grep '^rust/crates/sift_stream_bindings/src/' || true)) - -echo -echo "Changed SiftStream Python bindings files:" -echo " ${bindings_changed_files[*]}" -echo +bindings_changed_files=($(git diff --name-only --diff-filter=ACM @{upstream}... | grep '^rust/crates/sift_stream_bindings/src/' || true)) +echo "→ Rust binding files changed:" if [[ -n "$bindings_changed_files" ]]; then - echo "Rust bindings files changed, running Rust stub checks..." + echo " ${bindings_changed_files[*]}" + echo + echo " [1/1] Stub generation..." bash "$GITHOOKS_DIR/pre-push-rust/stubs.sh" + echo +else + echo " (none)" + echo fi -echo "Pre-push checks completed successfully." +echo "========================================" +echo " ✓ All checks passed" +echo "========================================" diff --git a/.githooks/pre-push-python/extras.sh b/.githooks/pre-push-python/extras.sh index 362e3a42a..b73b6c2a2 100755 --- a/.githooks/pre-push-python/extras.sh +++ b/.githooks/pre-push-python/extras.sh @@ -11,19 +11,20 @@ check_extras_changes() { local changed_files=$(git status --porcelain "$target_path" || true) if [ -n "$changed_files" ]; then - echo "❌ ERROR: Generated pyproject.toml extras are not up-to-date. Please commit the changed files:" - echo "$changed_files" + echo " ✗ ERROR: Generated extras are not up-to-date:" + echo "$changed_files" | sed 's/^/ /' + echo " Please commit these changes before pushing." exit 1 fi } # Function to generate Python extras generate_python_extras() { - echo "Generating Python extras..." + echo " → Generating extras..." cd "$PYTHON_DIR" if [[ ! -d "$PYTHON_DIR/venv" ]]; then - echo "Running bootstrap script..." + echo " → Running bootstrap script..." bash ./scripts/dev bootstrap fi @@ -33,4 +34,4 @@ generate_python_extras() { generate_python_extras -echo "All extras are up-to-date." +echo " ✓ Extras are up-to-date" diff --git a/.githooks/pre-push-python/fmt-lint.sh b/.githooks/pre-push-python/fmt-lint.sh index 5ff8994df..baef354b1 100644 --- a/.githooks/pre-push-python/fmt-lint.sh +++ b/.githooks/pre-push-python/fmt-lint.sh @@ -6,17 +6,15 @@ set -e REPO_ROOT="$(git rev-parse --show-toplevel)" PYTHON_DIR="$REPO_ROOT/python" -echo "Running Python formatting and linting with --fix..." - # Change to Python directory cd "$PYTHON_DIR" # Run ruff format (formatter) -echo "Running ruff format..." +echo " → Running ruff format..." bash ./scripts/dev fmt # Run ruff check with --fix (linter) -echo "Running ruff check --fix..." +echo " → Running ruff check --fix..." bash ./scripts/dev lint-fix # Check if any files were modified by formatting/linting @@ -25,11 +23,11 @@ changed_files=$(git status --porcelain python/lib/sift_client/ | grep -E '\.py$' if [ -n "$changed_files" ]; then echo "" - echo "❌ ERROR: Formatting/linting made changes to the following files:" - echo "$changed_files" + echo " ✗ ERROR: Formatting/linting made changes:" + echo "$changed_files" | sed 's/^/ /' echo "" - echo "Please commit these changes before pushing." + echo " Please commit these changes before pushing." exit 1 fi -echo "Python formatting and linting completed successfully." +echo " ✓ Formatting and linting passed" diff --git a/.githooks/pre-push-python/stubs.sh b/.githooks/pre-push-python/stubs.sh index 6efdab8f6..9208ffe43 100644 --- a/.githooks/pre-push-python/stubs.sh +++ b/.githooks/pre-push-python/stubs.sh @@ -11,19 +11,20 @@ check_stub_changes() { local changed_files=$(git status --porcelain "$target_path" | grep -E '\.pyi$' || true) if [ -n "$changed_files" ]; then - echo "❌ ERROR: Generated python stubs are not up-to-date. Please commit the changed files:" - echo "$changed_files" + echo " ✗ ERROR: Generated stubs are not up-to-date:" + echo "$changed_files" | sed 's/^/ /' + echo " Please commit these changes before pushing." exit 1 fi } # Function to generate Python stubs generate_python_stubs() { - echo "Generating Python stubs..." + echo " → Generating stubs..." cd "$PYTHON_DIR" if [[ ! -d "$PYTHON_DIR/venv" ]]; then - echo "Running bootstrap script..." + echo " → Running bootstrap script..." bash ./scripts/dev bootstrap fi @@ -33,4 +34,4 @@ generate_python_stubs() { generate_python_stubs -echo "All stubs are up-to-date." \ No newline at end of file +echo " ✓ Stubs are up-to-date" \ No newline at end of file diff --git a/.githooks/pre-push-rust/stubs.sh b/.githooks/pre-push-rust/stubs.sh index 1af106713..babf594e1 100644 --- a/.githooks/pre-push-rust/stubs.sh +++ b/.githooks/pre-push-rust/stubs.sh @@ -10,15 +10,16 @@ check_stub_changes() { local changed_files=$(git status --porcelain "$target_path" | grep -E '\.pyi$' || true) if [ -n "$changed_files" ]; then - echo "ERROR: Generated python stubs are not up-to-date. Please commit the changed files:" - echo "$changed_files" + echo " ✗ ERROR: Generated stubs are not up-to-date:" + echo "$changed_files" | sed 's/^/ /' + echo " Please commit these changes before pushing." exit 1 fi } # Function to generate bindings stubs generate_bindings_stubs() { - echo "Generating bindings stubs..." + echo " → Generating stubs..." cd "$BINDINGS_DIR" cargo run --bin stub_gen @@ -29,4 +30,4 @@ generate_bindings_stubs() { generate_bindings_stubs -echo "All stubs are up-to-date." \ No newline at end of file +echo " ✓ Stubs are up-to-date" \ No newline at end of file diff --git a/python/pyproject.toml b/python/pyproject.toml index 2e18b8199..d19f97772 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -118,8 +118,6 @@ development = [ 'tomlkit~=0.13.3', ] - - docs = [ 'griffe-pydantic', 'mike', From ebd7aa5bbe133fa341ea4a3d539edc4568ebf6e8 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 15:07:53 -0800 Subject: [PATCH 08/16] fix pre-push --- .githooks/pre-push | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.githooks/pre-push b/.githooks/pre-push index 0313d0275..cff188e15 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -4,6 +4,9 @@ REPO_ROOT="$(git rev-parse --show-toplevel)" GITHOOKS_DIR="$REPO_ROOT/.githooks" +# Track failures +FAILED=0 + echo "========================================" echo " Pre-Push Checks" echo "========================================" @@ -18,13 +21,13 @@ if [[ -n "$python_changed_files" ]]; then echo " ${python_changed_files[*]}" echo echo " [1/3] Formatting and linting..." - bash "$GITHOOKS_DIR/pre-push-python/fmt-lint.sh" + bash "$GITHOOKS_DIR/pre-push-python/fmt-lint.sh" || FAILED=1 echo " [2/3] Stub generation..." - bash "$GITHOOKS_DIR/pre-push-python/stubs.sh" + bash "$GITHOOKS_DIR/pre-push-python/stubs.sh" || FAILED=1 echo " [3/3] Extras validation..." - bash "$GITHOOKS_DIR/pre-push-python/extras.sh" + bash "$GITHOOKS_DIR/pre-push-python/extras.sh" || FAILED=1 echo else echo " (none)" @@ -39,14 +42,21 @@ if [[ -n "$bindings_changed_files" ]]; then echo " ${bindings_changed_files[*]}" echo echo " [1/1] Stub generation..." - bash "$GITHOOKS_DIR/pre-push-rust/stubs.sh" + bash "$GITHOOKS_DIR/pre-push-rust/stubs.sh" || FAILED=1 echo else echo " (none)" echo fi -echo "========================================" -echo " ✓ All checks passed" -echo "========================================" +if [[ $FAILED -eq 1 ]]; then + echo "========================================" + echo " ✗ Some checks failed" + echo "========================================" + exit 1 +else + echo "========================================" + echo " ✓ All checks passed" + echo "========================================" +fi From 83de1f9c31aa975d1d87019a64b780d6921f1ae8 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 15:08:30 -0800 Subject: [PATCH 09/16] test commit --- python/pyproject.toml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index d19f97772..8f494b242 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -118,16 +118,9 @@ development = [ 'tomlkit~=0.13.3', ] -docs = [ - 'griffe-pydantic', - 'mike', - 'mkdocs', - 'mkdocs-api-autonav', - 'mkdocs-include-markdown-plugin', - 'mkdocs-jupyter', - 'mkdocs-material', - 'mkdocstrings[python]', -] + + + file-imports = [ 'h5py~=3.11', 'npTDMS~=1.9', From 34c4bdd27f5dd06a3c45033866b5c63512a2b09b Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 15:09:57 -0800 Subject: [PATCH 10/16] commit after failed push --- .githooks/pre-push-python/extras.sh | 2 +- .githooks/pre-push-python/fmt-lint.sh | 2 +- .githooks/pre-push-python/stubs.sh | 2 +- .githooks/pre-push-rust/stubs.sh | 2 +- python/pyproject.toml | 34 +++++++++++++++++++++++---- 5 files changed, 34 insertions(+), 8 deletions(-) diff --git a/.githooks/pre-push-python/extras.sh b/.githooks/pre-push-python/extras.sh index b73b6c2a2..56cb00620 100755 --- a/.githooks/pre-push-python/extras.sh +++ b/.githooks/pre-push-python/extras.sh @@ -11,7 +11,7 @@ check_extras_changes() { local changed_files=$(git status --porcelain "$target_path" || true) if [ -n "$changed_files" ]; then - echo " ✗ ERROR: Generated extras are not up-to-date:" + echo " ❌ ERROR: Generated extras are not up-to-date:" echo "$changed_files" | sed 's/^/ /' echo " Please commit these changes before pushing." exit 1 diff --git a/.githooks/pre-push-python/fmt-lint.sh b/.githooks/pre-push-python/fmt-lint.sh index baef354b1..f112c7f09 100644 --- a/.githooks/pre-push-python/fmt-lint.sh +++ b/.githooks/pre-push-python/fmt-lint.sh @@ -23,7 +23,7 @@ changed_files=$(git status --porcelain python/lib/sift_client/ | grep -E '\.py$' if [ -n "$changed_files" ]; then echo "" - echo " ✗ ERROR: Formatting/linting made changes:" + echo " ❌ ERROR: Formatting/linting made changes:" echo "$changed_files" | sed 's/^/ /' echo "" echo " Please commit these changes before pushing." diff --git a/.githooks/pre-push-python/stubs.sh b/.githooks/pre-push-python/stubs.sh index 9208ffe43..9ee86fb42 100644 --- a/.githooks/pre-push-python/stubs.sh +++ b/.githooks/pre-push-python/stubs.sh @@ -11,7 +11,7 @@ check_stub_changes() { local changed_files=$(git status --porcelain "$target_path" | grep -E '\.pyi$' || true) if [ -n "$changed_files" ]; then - echo " ✗ ERROR: Generated stubs are not up-to-date:" + echo " ❌ ERROR: Generated stubs are not up-to-date:" echo "$changed_files" | sed 's/^/ /' echo " Please commit these changes before pushing." exit 1 diff --git a/.githooks/pre-push-rust/stubs.sh b/.githooks/pre-push-rust/stubs.sh index babf594e1..980750c9d 100644 --- a/.githooks/pre-push-rust/stubs.sh +++ b/.githooks/pre-push-rust/stubs.sh @@ -10,7 +10,7 @@ check_stub_changes() { local changed_files=$(git status --porcelain "$target_path" | grep -E '\.pyi$' || true) if [ -n "$changed_files" ]; then - echo " ✗ ERROR: Generated stubs are not up-to-date:" + echo " ❌ ERROR: Generated stubs are not up-to-date:" echo "$changed_files" | sed 's/^/ /' echo " Please commit these changes before pushing." exit 1 diff --git a/python/pyproject.toml b/python/pyproject.toml index 8f494b242..8a609f334 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -117,10 +117,36 @@ development = [ 'ruff~=0.12.10', 'tomlkit~=0.13.3', ] - - - - +docs = [ + 'griffe-pydantic', + 'mike', + 'mkdocs', + 'mkdocs-api-autonav', + 'mkdocs-include-markdown-plugin', + 'mkdocs-jupyter', + 'mkdocs-material', + 'mkdocstrings[python]', +] +docs-build = [ + 'griffe-pydantic', + 'grpcio-testing~=1.13', + 'mike', + 'mkdocs', + 'mkdocs-api-autonav', + 'mkdocs-include-markdown-plugin', + 'mkdocs-jupyter', + 'mkdocs-material', + 'mkdocstrings[python]', + 'mypy==1.10.0', + 'pyright==1.1.386', + 'pytest-asyncio==0.23.7', + 'pytest-benchmark==4.0.0', + 'pytest-dotenv==0.5.2', + 'pytest-mock==3.14.0', + 'pytest==8.2.2', + 'ruff~=0.12.10', + 'tomlkit~=0.13.3', +] file-imports = [ 'h5py~=3.11', 'npTDMS~=1.9', From 6f18e5e60e003a38436fa827bc70fcd1d9611746 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 15:12:56 -0800 Subject: [PATCH 11/16] update CI --- .github/workflows/python_ci.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/python_ci.yaml b/.github/workflows/python_ci.yaml index cde4ad58b..1e5818dc5 100644 --- a/.github/workflows/python_ci.yaml +++ b/.github/workflows/python_ci.yaml @@ -48,6 +48,16 @@ jobs: run: | pyright lib + - name: Check Stubs Generation + working-directory: . + run: | + bash .githooks/pre-push-python/stubs.sh + + - name: Check Extras Generation + working-directory: . + run: | + bash .githooks/pre-push-python/extras.sh + - name: Pytest Unit Tests run: | pytest -m "not integration" From a37bc855c16d7517622fd7aecda7ebed7fdc5ff2 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 15:19:48 -0800 Subject: [PATCH 12/16] update extras --- python/pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index 8a609f334..1a9f66851 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -55,6 +55,7 @@ all = [ 'npTDMS~=1.9', 'polars~=1.8', 'pyOpenSSL<24.0.0', + 'pyarrow>=17.0.0', 'rosbags~=0.0', 'sift-stream-bindings>=0.2.0-rc', 'types-pyOpenSSL<24.0.0', @@ -63,6 +64,9 @@ build = [ 'build==1.2.1', 'pdoc==14.5.0', ] +data-review = [ + 'pyarrow>=17.0.0', +] dev = [ 'grpcio-testing~=1.13', 'mypy==1.10.0', @@ -93,6 +97,7 @@ dev-all = [ 'pdoc==14.5.0', 'polars~=1.8', 'pyOpenSSL<24.0.0', + 'pyarrow>=17.0.0', 'pyright==1.1.386', 'pytest-asyncio==0.23.7', 'pytest-benchmark==4.0.0', From 42d91021a0e310260d5b80f8a5a07113d30b55de Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 15:29:50 -0800 Subject: [PATCH 13/16] update extras to exclude docs from dev-all since it requires newer python version --- python/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 1a9f66851..4aecc4687 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -225,8 +225,8 @@ sift-stream-bindings = ["sift-stream"] file-imports = ["tdms", "rosbags", "hdf5"] all = ["openssl", "sift-stream", "file-imports", "data-review"] -dev-all = ["development", "build", "docs", "all"] -docs-build = ["development", "docs"] +dev-all = ["development", "all"] +docs-build = ["dev-all", "docs"] # Note python 3.9+ [tool.mypy] python_version = "3.10" # Use the Python 3.10 type checker since we are using eval-type-backport and `from __future__ import annotations` From 09503b8cbcec5ecc4b0da92b3f1599bda382e207 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 15:34:06 -0800 Subject: [PATCH 14/16] add missing build --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 4aecc4687..17d3b0b44 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -225,7 +225,7 @@ sift-stream-bindings = ["sift-stream"] file-imports = ["tdms", "rosbags", "hdf5"] all = ["openssl", "sift-stream", "file-imports", "data-review"] -dev-all = ["development", "all"] +dev-all = ["development", "all", "build"] docs-build = ["dev-all", "docs"] # Note python 3.9+ [tool.mypy] From f220fa81530b53ffda43d158b926bc6430ba4802 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 12 Nov 2025 15:34:50 -0800 Subject: [PATCH 15/16] update extras --- python/pyproject.toml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 17d3b0b44..4dbc5664b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -82,16 +82,8 @@ dev = [ dev-all = [ 'build==1.2.1', 'cffi~=1.14', - 'griffe-pydantic', 'grpcio-testing~=1.13', 'h5py~=3.11', - 'mike', - 'mkdocs', - 'mkdocs-api-autonav', - 'mkdocs-include-markdown-plugin', - 'mkdocs-jupyter', - 'mkdocs-material', - 'mkdocstrings[python]', 'mypy==1.10.0', 'npTDMS~=1.9', 'pdoc==14.5.0', @@ -133,8 +125,11 @@ docs = [ 'mkdocstrings[python]', ] docs-build = [ + 'build==1.2.1', + 'cffi~=1.14', 'griffe-pydantic', 'grpcio-testing~=1.13', + 'h5py~=3.11', 'mike', 'mkdocs', 'mkdocs-api-autonav', @@ -143,14 +138,22 @@ docs-build = [ 'mkdocs-material', 'mkdocstrings[python]', 'mypy==1.10.0', + 'npTDMS~=1.9', + 'pdoc==14.5.0', + 'polars~=1.8', + 'pyOpenSSL<24.0.0', + 'pyarrow>=17.0.0', 'pyright==1.1.386', 'pytest-asyncio==0.23.7', 'pytest-benchmark==4.0.0', 'pytest-dotenv==0.5.2', 'pytest-mock==3.14.0', 'pytest==8.2.2', + 'rosbags~=0.0', 'ruff~=0.12.10', + 'sift-stream-bindings>=0.2.0-rc', 'tomlkit~=0.13.3', + 'types-pyOpenSSL<24.0.0', ] file-imports = [ 'h5py~=3.11', From 041e2fd39e4ea79609437bd8873581dc86829d7c Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Thu, 13 Nov 2025 11:22:41 -0800 Subject: [PATCH 16/16] add docs --- python/scripts/generate_extras.py | 37 +++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/python/scripts/generate_extras.py b/python/scripts/generate_extras.py index 3e4ec0272..731765571 100644 --- a/python/scripts/generate_extras.py +++ b/python/scripts/generate_extras.py @@ -1,3 +1,24 @@ +""" +Generate [project.optional-dependencies] from [tool.sift.extras] configuration. + +This script reads the custom [tool.sift.extras] section in pyproject.toml and generates +the standard [project.optional-dependencies] section automatically. + +Input sections: + - [tool.sift.extras]: Atomic extras with direct dependency lists + - [tool.sift.extras.combine]: Combined extras that reference other extras + +Output: + - [project.optional-dependencies]: Auto-generated, sorted extras with resolved dependencies + +The script: + 1. Reads atomic extras (e.g., "dev", "openssl") from [tool.sift.extras] + 2. Reads combined extras (e.g., "all" = ["openssl", "dev"]) from [tool.sift.extras.combine] + 3. Recursively resolves all dependencies + 4. Writes sorted, deduplicated lists to [project.optional-dependencies] + 5. Preserves TOML formatting and comments using tomlkit +""" + import sys from pathlib import Path @@ -20,9 +41,21 @@ atomic_extras = {k: v for k, v in tool_sift.items() if k != "combine"} -# Recursive resolver for nested combines def resolve(name, stack=None): - """Recursively resolve nested combine groups.""" + """ + Recursively resolve an extra's dependencies. + + Args: + name: The extra name to resolve (e.g., "dev-all") + stack: Internal tracking for cycle detection + + Returns: + List of all dependency strings for this extra + + Raises: + ValueError: If a cyclic dependency is detected + KeyError: If an unknown extra is referenced + """ if stack is None: stack = [] if name in stack: