diff --git a/.github/ci-test-config.json b/.github/ci-test-config.json new file mode 100644 index 0000000..e3ee2c1 --- /dev/null +++ b/.github/ci-test-config.json @@ -0,0 +1,71 @@ +{ + "py-io4dolfinx": { + "versions": ["main", "1.1.2"], + "specs": [ + "^py-fenics-dolfinx%c,cxx=gcc@13.3.0", + "+adios2+h5py+xdmf+vtkhdf+pyvista ^py-fenics-dolfinx%c,cxx=gcc@13.3.0" + ], + "import_name": "io4dolfinx", + "remote_repos": "https://github.com/FEniCS/spack-fenics.git" + }, + "py-meshio": { + "versions": ["5.3.5"], + "specs": ["", "+xdmf"], + "import_name": "meshio" + }, + "py-mri-toolkit": { + "versions": ["main", "0.2.0"], + "specs": ["", "+show"], + "import_name": "mritk" + }, + "py-mri2mesh": { + "versions": ["main", "0.3.0"], + "specs": ["", "+mesh"], + "import_name": "mri2mesh" + }, + "py-networks-fenicsx": { + "versions": ["main", "0.2.0"], + "specs": ["^py-fenics-dolfinx%gcc@13"], + "import_name": "networks_fenicsx" + }, + "py-scifem": { + "versions": ["main", "0.16.1", "0.14.0"], + "specs": ["+adios2+petsc+biomed+hdf5 ^petsc+mumps"], + "import_name": "scifem", + "remote_repos": "https://github.com/FEniCS/spack-fenics.git" + }, + "py-fenicsx-beat": { + "versions": ["main", "0.2.4"], + "specs": ["^py-fenics-dolfinx%gcc@13"], + "import_name": "beat" + }, + "py-gotranx": { + "versions": ["main", "1.5.1"], + "specs": [""] + }, + "py-circulation": { + "versions": ["main", "0.2.1"], + "specs": [""], + "import_name": "circulation" + }, + "py-cardiac-geometriesx": { + "versions": ["main", "0.12.0"], + "specs": ["^py-fenics-dolfinx%c,cxx=gcc@13.3.0"], + "import_name": "cardiac_geometries" + }, + "py-fenicsx-pulse": { + "versions": ["main", "0.6.0"], + "specs": ["^py-fenics-dolfinx%c,cxx=gcc@13.3.0"], + "import_name": "pulse" + }, + "py-ukb-atlas": { + "versions": ["main", "1.2.3"], + "specs": [""], + "import_name": "ukb" + }, + "py-fenicsx-ldrb": { + "versions": ["main", "0.1.17"], + "specs": ["^py-fenics-dolfinx%c,cxx=gcc@13.3.0"], + "import_name": "ldrb" + } +} diff --git a/.github/scripts/generate_matrix.py b/.github/scripts/generate_matrix.py new file mode 100644 index 0000000..4398c12 --- /dev/null +++ b/.github/scripts/generate_matrix.py @@ -0,0 +1,36 @@ +import json +import os + + +def generate_matrix(): + with open(".github/ci-test-config.json", "r") as f: + config = json.load(f) + + matrix_include = [] + for pkg_name, details in config.items(): + for version in details["versions"]: + for spec in details["specs"]: + # Import name defaults to the package name without 'py-' if not provided + import_name = details.get( + "import_name", pkg_name.replace("py-", "").replace("-", "_") + ) + matrix_include.append( + { + "package": pkg_name, + "version": version, + "spec": spec, + "import_name": import_name, + "remote_repos": details.get("remote_repos", ""), + } + ) + + # Write the output to GitHub Actions environment + if "GITHUB_OUTPUT" in os.environ: + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"matrix={json.dumps(matrix_include)}\n") + else: + print(json.dumps(matrix_include, indent=2)) + + +if __name__ == "__main__": + generate_matrix() diff --git a/.github/scripts/update_packages.py b/.github/scripts/update_packages.py new file mode 100644 index 0000000..0a23d84 --- /dev/null +++ b/.github/scripts/update_packages.py @@ -0,0 +1,162 @@ +import hashlib +import json +import os +import re +import urllib.request +from pathlib import Path + +# Token is provided by GitHub Actions to avoid rate limits +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") + + +def get_github_latest_release(owner, repo): + url = f"https://api.github.com/repos/{owner}/{repo}/tags" + req = urllib.request.Request(url) + if GITHUB_TOKEN: + req.add_header("Authorization", f"Bearer {GITHUB_TOKEN}") + try: + with urllib.request.urlopen(req) as response: + tags = json.loads(response.read().decode()) + if not tags: + return None, None + + # Filter for tags that look like semver (e.g., v1.0.0, 0.9.4) + valid_tags = [t["name"] for t in tags if re.match(r"^v?\d+\.\d+", t["name"])] + if not valid_tags: + return None, None + + # Sort to find the highest semver release + def parse_version(tag): + return [int(x) for x in re.findall(r"\d+", tag)] + + latest_tag = max(valid_tags, key=parse_version) + + # Spack versions typically drop the leading 'v' + spack_ver = latest_tag[1:] if latest_tag.startswith("v") else latest_tag + return spack_ver, latest_tag + except Exception as e: + print(f"Failed to fetch from GitHub for {owner}/{repo}: {e}") + return None, None + + +def get_pypi_latest_release(pypi_string): + # pypi_string example: "annotated-doc/annotated_doc-0.0.4.tar.gz" + pkg_name = pypi_string.split("/")[0] + url = f"https://pypi.org/pypi/{pkg_name}/json" + try: + with urllib.request.urlopen(url) as response: + data = json.loads(response.read().decode()) + latest_version = data["info"]["version"] + + # Find sdist sha256 checksum + for release in data["releases"].get(latest_version, []): + if release["packagetype"] == "sdist": + return latest_version, release["digests"]["sha256"] + return latest_version, None + except Exception as e: + print(f"Failed to fetch from PyPI for {pkg_name}: {e}") + return None, None + + +def compute_sha256(url): + try: + req = urllib.request.Request(url) + if "api.github.com" in url and GITHUB_TOKEN: + req.add_header("Authorization", f"Bearer {GITHUB_TOKEN}") + with urllib.request.urlopen(req) as response: + return hashlib.sha256(response.read()).hexdigest() + except Exception as e: + print(f"Failed to download and hash {url}: {e}") + return None + + +def process_package(package_py_path): + with open(package_py_path, "r") as f: + content = f.read() + + new_version = None + sha256 = None + + # Determine if package sources from PyPI or GitHub + pypi_match = re.search(r'pypi\s*=\s*["\']([^"\']+)["\']', content) + github_match = re.search( + r'(?:git|url)\s*=\s*["\']https://github\.com/([^/]+)/([^/]+?)(?:\.git|/archive.*)["\']', + content, + ) + + if pypi_match: + new_version, sha256 = get_pypi_latest_release(pypi_match.group(1)) + elif github_match: + owner, repo = github_match.groups() + new_version, tag_name = get_github_latest_release(owner, repo) + if new_version: + # Check if this version is already declared + if f'version("{new_version}"' in content or f"version('{new_version}'" in content: + return False + + # Get sha256 for the new tarball + tar_url = f"https://github.com/{owner}/{repo}/archive/refs/tags/{tag_name}.tar.gz" + sha256 = compute_sha256(tar_url) + + if not new_version or not sha256: + return False + + if f'version("{new_version}"' in content or f"version('{new_version}'" in content: + return False + + print(f"Found new version {new_version} for {package_py_path.parent.name}") + + # Inject the new version block right after the last version() block + lines = content.split("\n") + last_version_idx = -1 + for i, line in enumerate(lines): + if line.strip().startswith("version("): + last_version_idx = i + + if last_version_idx != -1: + new_line = f' version("{new_version}", sha256="{sha256}")' + lines.insert(last_version_idx + 1, new_line) + + with open(package_py_path, "w") as f: + f.write("\n".join(lines)) + + # Update CI matrix JSON so the new version gets tested automatically + ci_config_path = Path(".github/ci-test-config.json") + if ci_config_path.exists(): + with open(ci_config_path, "r") as f: + ci_config = json.load(f) + + # Spack packages use underscores (py_scifem) but in CI specs they use dashes (py-scifem) + spack_pkg_name = package_py_path.parent.name.replace("_", "-") + + if spack_pkg_name in ci_config: + if new_version not in ci_config[spack_pkg_name]["versions"]: + # Insert right after 'main' if it exists, otherwise at start + if "main" in ci_config[spack_pkg_name]["versions"]: + ci_config[spack_pkg_name]["versions"].insert(1, new_version) + else: + ci_config[spack_pkg_name]["versions"].insert(0, new_version) + + with open(ci_config_path, "w") as f: + json.dump(ci_config, f, indent=2) + print(f"Added {new_version} to CI matrix for {spack_pkg_name}") + + return True + + return False + + +if __name__ == "__main__": + repo_dir = Path("spack_repo/scientificcomputing/packages") + updates = 0 + for pkg_dir in repo_dir.iterdir(): + if pkg_dir.is_dir(): + pkg_file = pkg_dir / "package.py" + if pkg_file.exists(): + if process_package(pkg_file): + updates += 1 + + if updates > 0: + print(f"Successfully updated {updates} packages.") + else: + print("All packages are up to date.") diff --git a/.github/workflows/spack-dependabot.yml b/.github/workflows/spack-dependabot.yml new file mode 100644 index 0000000..500859d --- /dev/null +++ b/.github/workflows/spack-dependabot.yml @@ -0,0 +1,70 @@ +name: 📦 Spack Dependabot + +on: + schedule: + - cron: "0 2 * * MON" # Runs weekly on Mondays at 02:00 + workflow_dispatch: # Allows you to trigger it manually from the Actions tab + +permissions: + contents: write + pull-requests: write + +jobs: + spack-dependabot: + name: Check for Spack Package Updates + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Run Package Updater Script + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python .github/scripts/update_packages.py + + - name: Format with Ruff + run: | + pip install ruff + # Aligns the newly injected `version()` blocks with your ruff.toml + ruff format spack_repo/scientificcomputing/packages/ + + - name: Commit and Create Pull Request + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Check if there are any changes (if not, we can exit early) + if [[ -z $(git status -s) ]]; then + echo "No updates found. Skipping PR creation." + exit 0 + fi + + # Configure standard GitHub Actions bot git credentials + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # Create a dynamic branch name based on the current timestamp + BRANCH_NAME="spack-dependabot-updates-$(date +%Y%m%d-%H%M%S)" + git checkout -b "$BRANCH_NAME" + + # Stage and commit the formatted updates + git add spack_repo/scientificcomputing/packages/ .github/ci-test-config.json + git commit -m "chore(deps): update spack packages to latest upstream versions" + + # Push the new branch to the repository + git push origin "$BRANCH_NAME" + + # Use the native GitHub CLI to create the pull request + gh pr create \ + --title "Bump Spack packages to latest upstream versions" \ + --body "### Spack Dependabot Updates 🤖 + This is an automated PR updating your \`package.py\` definitions and CI testing matrix to their latest upstream versions (discovered via the GitHub API and PyPI)." \ + --label "new-package" \ + --head "$BRANCH_NAME" \ + --base main diff --git a/.github/workflows/spack-matrix.yml b/.github/workflows/spack-matrix.yml index a6f6bc9..7a51dba 100644 --- a/.github/workflows/spack-matrix.yml +++ b/.github/workflows/spack-matrix.yml @@ -1,4 +1,4 @@ -name: 🤖 +name: 🤖 CI Matrix on: push: @@ -19,7 +19,7 @@ concurrency: jobs: style-check: - name: ✨ + name: ✨ Style Check runs-on: ubuntu-latest steps: @@ -42,379 +42,48 @@ jobs: - run: chmod +x style_check.sh - run: ./style_check.sh HEAD^ - # py-dolfinx-mpc: - # needs: style-check - # runs-on: ubuntu-latest - # container: ubuntu:24.04 - # strategy: - # fail-fast: false - # matrix: - # version: ["main", "0.10.1", "0.9.3", "0.8.1"] - # spec: ["^py-petsc4py%gcc@13", "+numba ^py-petsc4py%gcc@13"] - - # steps: - # - uses: actions/checkout@v6 - # with: - # path: spack_repos - # - uses: ./spack_repos/.github/actions/test-package - # with: - # spec: py-dolfinx-mpc@${{ matrix.version }} ${{ matrix.spec }} - # remote_repos: "https://github.com/FEniCS/spack-fenics.git" - - # - name: Import test - # shell: spack-bash {0} - # run: | - # spack env activate ./env - # python -c "import dolfinx_mpc" - - # py-fenicsx-ii: - # needs: style-check - # runs-on: ubuntu-latest - # container: ubuntu:24.04 - # strategy: - # fail-fast: false - # matrix: - # version: ["main", "0.4.0"] - # spec: ["", "+petsc"] - - # steps: - # - uses: actions/checkout@v6 - # with: - # path: spack_repos - # - uses: ./spack_repos/.github/actions/test-package - # with: - # spec: py-fenicsx-ii@${{ matrix.version }} ${{ matrix.spec }} - # remote_repos: "https://github.com/FEniCS/spack-fenics.git" - - # - name: Import test - # shell: spack-bash {0} - # run: | - # spack env activate ./env - # python -c "import fenicsx_ii" - - py-io4dolfinx: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "1.1.2"] - spec: - [ - "^py-fenics-dolfinx%c,cxx=gcc@13.3.0", - "+adios2+h5py+xdmf+vtkhdf+pyvista ^py-fenics-dolfinx%c,cxx=gcc@13.3.0", - ] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-io4dolfinx@${{ matrix.version }} ${{ matrix.spec }} - remote_repos: "https://github.com/FEniCS/spack-fenics.git" - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import io4dolfinx" - - py-meshio: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["5.3.5"] - spec: ["", "+xdmf"] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-meshio@${{ matrix.version }} ${{ matrix.spec }} - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import meshio" - - py-mri-toolkit: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "0.2.0"] - spec: ["", "+show"] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-mri-toolkit@${{ matrix.version }} ${{ matrix.spec }} - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import mritk" - - py-mri2mesh: - needs: style-check + generate-matrix: + name: ⚙️ Generate Test Matrix runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "0.3.0"] - spec: ["", "+mesh"] - + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-mri2mesh@${{ matrix.version }} ${{ matrix.spec }} + - uses: actions/checkout@v4 + - name: Generate Matrix + id: set-matrix + run: python .github/scripts/generate_matrix.py - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import mri2mesh" - - py-networks-fenicsx: - needs: style-check + test-package: + needs: [style-check, generate-matrix] runs-on: ubuntu-latest container: ubuntu:24.04 strategy: fail-fast: false matrix: - version: ["main", "0.2.0"] - spec: ["^py-fenics-dolfinx%gcc@13"] - + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} + + name: ${{ matrix.package }}@${{ matrix.version }} ${{ matrix.spec }} steps: - uses: actions/checkout@v6 with: path: spack_repos - uses: ./spack_repos/.github/actions/test-package with: - spec: py-networks-fenicsx@${{ matrix.version }} ${{ matrix.spec }} + spec: ${{ matrix.package }}@${{ matrix.version }} ${{ matrix.spec }} + remote_repos: ${{ matrix.remote_repos }} - name: Import test shell: spack-bash {0} run: | spack env activate ./env - python -c "import networks_fenicsx" + python -c "import ${{ matrix.import_name }}" - py-scifem: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "0.16.1", "0.14.0"] - spec: ["+adios2+petsc+biomed+hdf5 ^petsc+mumps"] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-scifem@${{ matrix.version }} ${{ matrix.spec }} - remote_repos: "https://github.com/FEniCS/spack-fenics.git" - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import scifem" - - - name: Run tests + - name: Run tests (scifem only) + if: matrix.package == 'py-scifem' shell: spack-bash {0} run: | spack env activate ./env spack install --add py-pytest - spack stage py-scifem@${{ matrix.version }} ${{ matrix.spec }} - spack cd py-scifem@${{ matrix.version }} ${{ matrix.spec }} + spack stage ${{ matrix.package }}@${{ matrix.version }} ${{ matrix.spec }} + spack cd ${{ matrix.package }}@${{ matrix.version }} ${{ matrix.spec }} python -m pytest - - py-fenicsx-beat: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "0.2.4"] - spec: ["^py-fenics-dolfinx%gcc@13"] - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-fenicsx-beat@${{ matrix.version }} ${{ matrix.spec }} - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import beat" - - py-gotranx: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "1.5.1"] - spec: [""] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-gotranx@${{ matrix.version }} ${{ matrix.spec }} - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import gotranx" - - py-circulation: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "0.2.1"] - spec: [""] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-circulation@${{ matrix.version }} ${{ matrix.spec }} - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import circulation" - - py-cardiac-geometriesx: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "0.12.0"] - spec: ["^py-fenics-dolfinx%c,cxx=gcc@13.3.0"] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-cardiac-geometriesx@${{ matrix.version }} ${{ matrix.spec }} - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import cardiac_geometries" - - py-fenicsx-pulse: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "0.6.0"] - spec: ["^py-fenics-dolfinx%c,cxx=gcc@13.3.0"] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-fenicsx-pulse@${{ matrix.version }} ${{ matrix.spec }} - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import pulse" - - py-ukb-atlas: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "1.2.3"] - spec: [""] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-ukb-atlas@${{ matrix.version }} ${{ matrix.spec }} - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import ukb" - - - py-fenicsx-ldrb: - needs: style-check - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - fail-fast: false - matrix: - version: ["main", "0.1.17"] - spec: ["^py-fenics-dolfinx%c,cxx=gcc@13.3.0"] - - steps: - - uses: actions/checkout@v6 - with: - path: spack_repos - - uses: ./spack_repos/.github/actions/test-package - with: - spec: py-fenicsx-ldrb@${{ matrix.version }} ${{ matrix.spec }} - - - name: Import test - shell: spack-bash {0} - run: | - spack env activate ./env - python -c "import ldrb"