diff --git a/.githooks/pre-push b/.githooks/pre-push index ba7f5ac86..cff188e15 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -4,24 +4,59 @@ REPO_ROOT="$(git rev-parse --show-toplevel)" GITHOOKS_DIR="$REPO_ROOT/.githooks" +# Track failures +FAILED=0 + +echo "========================================" +echo " Pre-Push Checks" +echo "========================================" +echo + # Check for changes in Python files -python_changed_files=($(git diff --name-only --diff-filter=ACM | grep '^python/lib/sift_client/' || true)) +# Compare against the remote branch being pushed to +python_changed_files=($(git diff --name-only --diff-filter=ACM @{upstream}... | grep '^python/' || true)) +echo "→ Python files changed:" if [[ -n "$python_changed_files" ]]; then - echo "Python files changed, running Python formatting and linting..." - bash "$GITHOOKS_DIR/pre-push-python/fmt-lint.sh" + echo " ${python_changed_files[*]}" + echo + echo " [1/3] Formatting and linting..." + bash "$GITHOOKS_DIR/pre-push-python/fmt-lint.sh" || FAILED=1 - echo "Running Python stub checks..." - bash "$GITHOOKS_DIR/pre-push-python/stubs.sh" + echo " [2/3] Stub generation..." + bash "$GITHOOKS_DIR/pre-push-python/stubs.sh" || FAILED=1 + + echo " [3/3] Extras validation..." + bash "$GITHOOKS_DIR/pre-push-python/extras.sh" || FAILED=1 + echo +else + echo " (none)" + echo fi # Check for changes in Rust bindings files -bindings_changed_files=($(git diff --name-only --diff-filter=ACM | grep '^rust/crates/sift_stream_bindings/src/' || true)) +bindings_changed_files=($(git diff --name-only --diff-filter=ACM @{upstream}... | grep '^rust/crates/sift_stream_bindings/src/' || true)) +echo "→ Rust binding files changed:" if [[ -n "$bindings_changed_files" ]]; then - echo "Rust bindings files changed, running Rust stub checks..." - bash "$GITHOOKS_DIR/pre-push-rust/stubs.sh" + echo " ${bindings_changed_files[*]}" + echo + echo " [1/1] Stub generation..." + bash "$GITHOOKS_DIR/pre-push-rust/stubs.sh" || FAILED=1 + echo +else + echo " (none)" + echo fi -echo "Pre-push checks completed successfully." +if [[ $FAILED -eq 1 ]]; then + echo "========================================" + echo " ✗ Some checks failed" + echo "========================================" + exit 1 +else + echo "========================================" + echo " ✓ All checks passed" + echo "========================================" +fi diff --git a/.githooks/pre-push-python/extras.sh b/.githooks/pre-push-python/extras.sh new file mode 100755 index 000000000..56cb00620 --- /dev/null +++ b/.githooks/pre-push-python/extras.sh @@ -0,0 +1,37 @@ +# ensure generated pyproject.toml extras are up-to-date + +# Store the root directory of the repository +REPO_ROOT="$(git rev-parse --show-toplevel)" +PYTHON_DIR="$REPO_ROOT/python" +PYPROJECT_FILE="$PYTHON_DIR/pyproject.toml" + +# Function to check if pyproject.toml has changed +check_extras_changes() { + local target_path="$1" + local changed_files=$(git status --porcelain "$target_path" || true) + + if [ -n "$changed_files" ]; then + echo " ❌ ERROR: Generated extras are not up-to-date:" + echo "$changed_files" | sed 's/^/ /' + echo " Please commit these changes before pushing." + exit 1 + fi +} + +# Function to generate Python extras +generate_python_extras() { + echo " → Generating extras..." + cd "$PYTHON_DIR" + + if [[ ! -d "$PYTHON_DIR/venv" ]]; then + echo " → Running bootstrap script..." + bash ./scripts/dev bootstrap + fi + + bash ./scripts/dev gen-extras + check_extras_changes "$PYPROJECT_FILE" +} + +generate_python_extras + +echo " ✓ Extras are up-to-date" diff --git a/.githooks/pre-push-python/fmt-lint.sh b/.githooks/pre-push-python/fmt-lint.sh index 575a4475b..f112c7f09 100644 --- a/.githooks/pre-push-python/fmt-lint.sh +++ b/.githooks/pre-push-python/fmt-lint.sh @@ -6,17 +6,15 @@ set -e REPO_ROOT="$(git rev-parse --show-toplevel)" PYTHON_DIR="$REPO_ROOT/python" -echo "Running Python formatting and linting with --fix..." - # Change to Python directory cd "$PYTHON_DIR" # Run ruff format (formatter) -echo "Running ruff format..." +echo " → Running ruff format..." bash ./scripts/dev fmt # Run ruff check with --fix (linter) -echo "Running ruff check --fix..." +echo " → Running ruff check --fix..." bash ./scripts/dev lint-fix # Check if any files were modified by formatting/linting @@ -25,11 +23,11 @@ changed_files=$(git status --porcelain python/lib/sift_client/ | grep -E '\.py$' if [ -n "$changed_files" ]; then echo "" - echo "ERROR: Formatting/linting made changes to the following files:" - echo "$changed_files" + echo " ❌ ERROR: Formatting/linting made changes:" + echo "$changed_files" | sed 's/^/ /' echo "" - echo "Please commit these changes before pushing." + echo " Please commit these changes before pushing." exit 1 fi -echo "Python formatting and linting completed successfully." +echo " ✓ Formatting and linting passed" diff --git a/.githooks/pre-push-python/stubs.sh b/.githooks/pre-push-python/stubs.sh index 42d6d712b..9ee86fb42 100644 --- a/.githooks/pre-push-python/stubs.sh +++ b/.githooks/pre-push-python/stubs.sh @@ -11,19 +11,20 @@ check_stub_changes() { local changed_files=$(git status --porcelain "$target_path" | grep -E '\.pyi$' || true) if [ -n "$changed_files" ]; then - echo "ERROR: Generated python stubs are not up-to-date. Please commit the changed files:" - echo "$changed_files" + echo " ❌ ERROR: Generated stubs are not up-to-date:" + echo "$changed_files" | sed 's/^/ /' + echo " Please commit these changes before pushing." exit 1 fi } # Function to generate Python stubs generate_python_stubs() { - echo "Generating Python stubs..." + echo " → Generating stubs..." cd "$PYTHON_DIR" if [[ ! -d "$PYTHON_DIR/venv" ]]; then - echo "Running bootstrap script..." + echo " → Running bootstrap script..." bash ./scripts/dev bootstrap fi @@ -33,4 +34,4 @@ generate_python_stubs() { generate_python_stubs -echo "All stubs are up-to-date." \ No newline at end of file +echo " ✓ Stubs are up-to-date" \ No newline at end of file diff --git a/.githooks/pre-push-rust/stubs.sh b/.githooks/pre-push-rust/stubs.sh index 1af106713..980750c9d 100644 --- a/.githooks/pre-push-rust/stubs.sh +++ b/.githooks/pre-push-rust/stubs.sh @@ -10,15 +10,16 @@ check_stub_changes() { local changed_files=$(git status --porcelain "$target_path" | grep -E '\.pyi$' || true) if [ -n "$changed_files" ]; then - echo "ERROR: Generated python stubs are not up-to-date. Please commit the changed files:" - echo "$changed_files" + echo " ❌ ERROR: Generated stubs are not up-to-date:" + echo "$changed_files" | sed 's/^/ /' + echo " Please commit these changes before pushing." exit 1 fi } # Function to generate bindings stubs generate_bindings_stubs() { - echo "Generating bindings stubs..." + echo " → Generating stubs..." cd "$BINDINGS_DIR" cargo run --bin stub_gen @@ -29,4 +30,4 @@ generate_bindings_stubs() { generate_bindings_stubs -echo "All stubs are up-to-date." \ No newline at end of file +echo " ✓ Stubs are up-to-date" \ No newline at end of file diff --git a/.github/workflows/python_build_docs.yml b/.github/workflows/python_build_docs.yml index 912cb3d33..a654ea3d7 100644 --- a/.github/workflows/python_build_docs.yml +++ b/.github/workflows/python_build_docs.yml @@ -2,7 +2,7 @@ name: Build and Deploy Python Docs (Dev) on: pull_request: - types: [opened, synchronize, closed] + types: [ opened, synchronize, closed ] paths: - 'python/docs/**' - 'python/lib/**' @@ -36,7 +36,7 @@ jobs: - name: Install dependencies run: | cd python - pip install -e .[docs,development] + pip install -e .[docs-build] - name: Extract version id: version @@ -91,8 +91,8 @@ jobs: - name: Install dependencies run: | cd python - pip install -e .[docs,development] - + pip install -e .[docs-build] + - name: Fetch gh-pages branch run: git fetch origin gh-pages --depth=1 diff --git a/.github/workflows/python_ci.yaml b/.github/workflows/python_ci.yaml index 2bb3579c7..1e5818dc5 100644 --- a/.github/workflows/python_ci.yaml +++ b/.github/workflows/python_ci.yaml @@ -30,7 +30,7 @@ jobs: id: install run: | python -m pip install --upgrade pip - pip install '.[development,data_review,openssl,tdms,rosbags,hdf5,sift-stream]' + pip install '.[dev-all]' - name: Lint run: | @@ -48,6 +48,16 @@ jobs: run: | pyright lib + - name: Check Stubs Generation + working-directory: . + run: | + bash .githooks/pre-push-python/stubs.sh + + - name: Check Extras Generation + working-directory: . + run: | + bash .githooks/pre-push-python/extras.sh + - name: Pytest Unit Tests run: | pytest -m "not integration" diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml index fad61f3d7..e37f65f80 100644 --- a/.github/workflows/python_release.yaml +++ b/.github/workflows/python_release.yaml @@ -17,13 +17,13 @@ jobs: publish-to-pypi: name: Upload release to PyPI - needs: [python-ci, build-offline-archives] + needs: [ python-ci, build-offline-archives ] runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/p/sift_py permissions: - id-token: write + id-token: write steps: - name: Download distributions uses: actions/download-artifact@v4 @@ -38,7 +38,7 @@ jobs: create-github-release: name: Create GitHub Release - needs: [build-offline-archives] + needs: [ build-offline-archives ] runs-on: ubuntu-latest permissions: contents: write @@ -110,7 +110,7 @@ jobs: - name: Install dependencies run: | cd python - pip install -e .[docs,development] + pip install -e .[docs-build] - name: Extract version and check if stable id: version @@ -125,7 +125,7 @@ jobs: MINOR=${BASH_REMATCH[2]} SUFFIX=${BASH_REMATCH[3]} VERSION="v${MAJOR}.${MINOR}" - + # Check if this is a stable release (no suffix like -alpha, -beta, -rc) if [[ -z "$SUFFIX" ]]; then # Stable release - use 'latest' alias and make visible diff --git a/python/pyproject.toml b/python/pyproject.toml index a490f6699..4dbc5664b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + [project] name = "sift_stack_py" version = "0.9.1" @@ -44,6 +48,144 @@ Repository = "https://github.com/sift-stack/sift/tree/main/python" Changelog = "https://github.com/sift-stack/sift/tree/main/python/CHANGELOG.md" [project.optional-dependencies] +# AUTO GENERATED EXTRAS — EDIT [tool.sift.extras] ONLY +all = [ + 'cffi~=1.14', + 'h5py~=3.11', + 'npTDMS~=1.9', + 'polars~=1.8', + 'pyOpenSSL<24.0.0', + 'pyarrow>=17.0.0', + 'rosbags~=0.0', + 'sift-stream-bindings>=0.2.0-rc', + 'types-pyOpenSSL<24.0.0', +] +build = [ + 'build==1.2.1', + 'pdoc==14.5.0', +] +data-review = [ + 'pyarrow>=17.0.0', +] +dev = [ + 'grpcio-testing~=1.13', + 'mypy==1.10.0', + 'pyright==1.1.386', + 'pytest-asyncio==0.23.7', + 'pytest-benchmark==4.0.0', + 'pytest-dotenv==0.5.2', + 'pytest-mock==3.14.0', + 'pytest==8.2.2', + 'ruff~=0.12.10', + 'tomlkit~=0.13.3', +] +dev-all = [ + 'build==1.2.1', + 'cffi~=1.14', + 'grpcio-testing~=1.13', + 'h5py~=3.11', + 'mypy==1.10.0', + 'npTDMS~=1.9', + 'pdoc==14.5.0', + 'polars~=1.8', + 'pyOpenSSL<24.0.0', + 'pyarrow>=17.0.0', + 'pyright==1.1.386', + 'pytest-asyncio==0.23.7', + 'pytest-benchmark==4.0.0', + 'pytest-dotenv==0.5.2', + 'pytest-mock==3.14.0', + 'pytest==8.2.2', + 'rosbags~=0.0', + 'ruff~=0.12.10', + 'sift-stream-bindings>=0.2.0-rc', + 'tomlkit~=0.13.3', + 'types-pyOpenSSL<24.0.0', +] +development = [ + 'grpcio-testing~=1.13', + 'mypy==1.10.0', + 'pyright==1.1.386', + 'pytest-asyncio==0.23.7', + 'pytest-benchmark==4.0.0', + 'pytest-dotenv==0.5.2', + 'pytest-mock==3.14.0', + 'pytest==8.2.2', + 'ruff~=0.12.10', + 'tomlkit~=0.13.3', +] +docs = [ + 'griffe-pydantic', + 'mike', + 'mkdocs', + 'mkdocs-api-autonav', + 'mkdocs-include-markdown-plugin', + 'mkdocs-jupyter', + 'mkdocs-material', + 'mkdocstrings[python]', +] +docs-build = [ + 'build==1.2.1', + 'cffi~=1.14', + 'griffe-pydantic', + 'grpcio-testing~=1.13', + 'h5py~=3.11', + 'mike', + 'mkdocs', + 'mkdocs-api-autonav', + 'mkdocs-include-markdown-plugin', + 'mkdocs-jupyter', + 'mkdocs-material', + 'mkdocstrings[python]', + 'mypy==1.10.0', + 'npTDMS~=1.9', + 'pdoc==14.5.0', + 'polars~=1.8', + 'pyOpenSSL<24.0.0', + 'pyarrow>=17.0.0', + 'pyright==1.1.386', + 'pytest-asyncio==0.23.7', + 'pytest-benchmark==4.0.0', + 'pytest-dotenv==0.5.2', + 'pytest-mock==3.14.0', + 'pytest==8.2.2', + 'rosbags~=0.0', + 'ruff~=0.12.10', + 'sift-stream-bindings>=0.2.0-rc', + 'tomlkit~=0.13.3', + 'types-pyOpenSSL<24.0.0', +] +file-imports = [ + 'h5py~=3.11', + 'npTDMS~=1.9', + 'polars~=1.8', + 'rosbags~=0.0', +] +hdf5 = [ + 'h5py~=3.11', + 'polars~=1.8', +] +openssl = [ + 'cffi~=1.14', + 'pyOpenSSL<24.0.0', + 'types-pyOpenSSL<24.0.0', +] +rosbags = [ + 'rosbags~=0.0', +] +sift-stream = [ + 'sift-stream-bindings>=0.2.0-rc', +] +sift-stream-bindings = [ + 'sift-stream-bindings>=0.2.0-rc', +] +tdms = [ + 'npTDMS~=1.9', +] + +[tool.sift.extras] +# Extras configurations defined here will be available in [project.optional-dependencies] + # development development = [ "grpcio-testing~=1.13", @@ -55,6 +197,7 @@ development = [ "pytest-mock==3.14.0", "pytest-dotenv==0.5.2", "ruff~=0.12.10", + "tomlkit~=0.13.3" ] build = ["pdoc==14.5.0", "build==1.2.1"] docs = ["mkdocs", @@ -72,12 +215,21 @@ tdms = ["npTDMS~=1.9"] rosbags = ["rosbags~=0.0"] sift-stream = ["sift-stream-bindings>=0.2.0-rc"] hdf5 = ["h5py~=3.11", "polars~=1.8"] -data_review = ["pyarrow>=17.0.0"] -# Ensure any new user build extras are added to .github/workflows/python_build.yaml +data-review = ["pyarrow>=17.0.0"] -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +[tool.sift.extras.combine] +# Combined extras configurations and aliases can be defined here + +# aliases +dev = ["development"] +sift-stream-bindings = ["sift-stream"] + +# combinations +file-imports = ["tdms", "rosbags", "hdf5"] +all = ["openssl", "sift-stream", "file-imports", "data-review"] + +dev-all = ["development", "all", "build"] +docs-build = ["dev-all", "docs"] # Note python 3.9+ [tool.mypy] python_version = "3.10" # Use the Python 3.10 type checker since we are using eval-type-backport and `from __future__ import annotations` diff --git a/python/scripts/build_utils.py b/python/scripts/build_utils.py index 43225ac8e..cfa085a25 100644 --- a/python/scripts/build_utils.py +++ b/python/scripts/build_utils.py @@ -104,7 +104,9 @@ def main(): # Get all extras from the wheel extras = get_extras_from_wheel(str(wheel_file)) - combinations = get_extra_combinations(extras, exclude=["development", "docs"]) + combinations = get_extra_combinations( + extras, exclude=["development", "docs", "dev", "dev-all", "docs-build"] + ) # Test base installation first test_install( diff --git a/python/scripts/dev b/python/scripts/dev index 7a95a5ced..6a610805b 100755 --- a/python/scripts/dev +++ b/python/scripts/dev @@ -20,6 +20,7 @@ Subcommands: pyright Runs 'pyright lib' for type checking check Runs all python linting checks gen-stubs Generates pyi stubs for sift_client synchronous wrappers + gen-extras Generates optional-dependencies in pyproject.toml mypy-stubs Runs stubtest (mypy) on the generated pyi stubs pip-install Install project dependencies test Execute tests @@ -34,7 +35,7 @@ EOT pip_install() { source venv/bin/activate - pip install '.[development,build,docs,openssl,tdms,rosbags,sift-stream,hdf5]' + pip install '.[all-dev]' pip install -e . } @@ -119,6 +120,13 @@ mypy_stubs() { stubtest sift_client.resources.sync_stubs } +gen_extras() { + source venv/bin/activate + cd scripts + python3 generate_extras.py + cd .. +} + shift "$((OPTIND - 1))" case "$1" in @@ -168,6 +176,9 @@ case "$1" in gen-stubs) gen_stubs ;; + gen-extras) + gen_extras + ;; mypy-stubs) mypy_stubs ;; diff --git a/python/scripts/generate_extras.py b/python/scripts/generate_extras.py new file mode 100644 index 000000000..731765571 --- /dev/null +++ b/python/scripts/generate_extras.py @@ -0,0 +1,104 @@ +""" +Generate [project.optional-dependencies] from [tool.sift.extras] configuration. + +This script reads the custom [tool.sift.extras] section in pyproject.toml and generates +the standard [project.optional-dependencies] section automatically. + +Input sections: + - [tool.sift.extras]: Atomic extras with direct dependency lists + - [tool.sift.extras.combine]: Combined extras that reference other extras + +Output: + - [project.optional-dependencies]: Auto-generated, sorted extras with resolved dependencies + +The script: + 1. Reads atomic extras (e.g., "dev", "openssl") from [tool.sift.extras] + 2. Reads combined extras (e.g., "all" = ["openssl", "dev"]) from [tool.sift.extras.combine] + 3. Recursively resolves all dependencies + 4. Writes sorted, deduplicated lists to [project.optional-dependencies] + 5. Preserves TOML formatting and comments using tomlkit +""" + +import sys +from pathlib import Path + +import tomlkit + +pyproject = Path("../pyproject.toml") +if not pyproject.exists(): + sys.exit(f"❌ No pyproject.toml found at {pyproject.resolve()}") + +# Parse preserving comments and formatting +doc = tomlkit.parse(pyproject.read_text()) + +try: + tool_sift = doc["tool"]["sift"]["extras"] +except KeyError: + sys.exit("❌ No [tool.sift.extras] section found in pyproject.toml") + +# Split atomic and combined definitions +combine_section = tool_sift.get("combine", {}) +atomic_extras = {k: v for k, v in tool_sift.items() if k != "combine"} + + +def resolve(name, stack=None): + """ + Recursively resolve an extra's dependencies. + + Args: + name: The extra name to resolve (e.g., "dev-all") + stack: Internal tracking for cycle detection + + Returns: + List of all dependency strings for this extra + + Raises: + ValueError: If a cyclic dependency is detected + KeyError: If an unknown extra is referenced + """ + if stack is None: + stack = [] + if name in stack: + raise ValueError(f"Cyclic combine detected: {' -> '.join(stack + [name])}") + if name in atomic_extras: + return list(atomic_extras[name]) + if name in combine_section: + deps = [] + for sub in combine_section[name]: + deps.extend(resolve(sub, stack + [name])) + return deps + raise KeyError(f"Unknown group '{name}' referenced in combine") + + +# Build final extras dictionary +final_extras = {} +for name in list(atomic_extras) + list(combine_section): + deps = resolve(name) + final_extras[name] = sorted(set(deps)) + +# Inject into [project.optional-dependencies] +project = doc.setdefault("project", tomlkit.table()) + +# Create the optional-dependencies table +opt_table = tomlkit.table() + +# Add a header comment BEFORE the section +opt_table.trivia.indent = "" +opt_table.trivia.comment = "\n# AUTO GENERATED EXTRAS — EDIT [tool.sift.extras] ONLY" + +# Write arrays in sorted order +for name in sorted(final_extras): + deps = final_extras[name] + arr = tomlkit.array(deps) + arr.multiline(True) + arr.as_string() + opt_table[name] = arr + + +# Assign back to project +project["optional-dependencies"] = opt_table + +# Dump back to file +pyproject.write_text(tomlkit.dumps(doc)) + +print("Updated [project.optional-dependencies]")