From d436afc91157e51fe105b21cd954aec16c762745 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 2 Jun 2025 09:52:01 -0500 Subject: [PATCH 01/14] ci: Add a note about architecture --- .github/workflows/nf-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 51e27840dbff..88b9be9d53e6 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -64,6 +64,7 @@ jobs: n_parents: 0 exclude_tags: "gpu" + # FIXME Can't we just do this with nf-test --filter? - name: Separate modules and subworkflows id: components run: | From 2c8987aaa58e758f8afa12243a5585995f512023 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 2 Jun 2025 10:55:23 -0500 Subject: [PATCH 02/14] feat: Add scripts for managing ARM build failures and filtering tests - Introduced `filter_arm_failure_tests.py` to filter test paths based on ARM build failure markers. - Added `manage_arm_failures.py` for adding, removing, and listing ARM failure markers for modules. - Created `report_arm_build_failure.py` to filter ARM builds based on failure markers in CI workflows. - Updated GitHub Actions workflows to integrate ARM failure filtering. - Documented ARM build failure management in `ARM_BUILD_FAILURES.md`. - Added example failure marker for `fastqc` module. --- .github/scripts/filter_arm_failure_tests.py | 124 +++++++++++ .github/scripts/manage_arm_failures.py | 163 ++++++++++++++ .github/scripts/report_arm_build_failure.py | 120 +++++++++++ .github/workflows/nf-test.yml | 25 ++- .github/workflows/wave.yml | 81 +++++-- docs/ARM_BUILD_FAILURES.md | 200 ++++++++++++++++++ modules/nf-core/fastqc/.arm-build-failure.yml | 7 + .../nf-core/kraken2/build/tests/main.nf.test | 4 +- scripts/manage-arm-failures.py | 176 +++++++++++++++ 9 files changed, 879 insertions(+), 21 deletions(-) create mode 100755 .github/scripts/filter_arm_failure_tests.py create mode 100755 .github/scripts/manage_arm_failures.py create mode 100755 .github/scripts/report_arm_build_failure.py create mode 100644 docs/ARM_BUILD_FAILURES.md create mode 100644 modules/nf-core/fastqc/.arm-build-failure.yml create mode 100644 scripts/manage-arm-failures.py diff --git a/.github/scripts/filter_arm_failure_tests.py b/.github/scripts/filter_arm_failure_tests.py new file mode 100755 index 000000000000..e3682767700a --- /dev/null +++ b/.github/scripts/filter_arm_failure_tests.py @@ -0,0 +1,124 @@ +#!/usr/bin/env uv run +# /// script +# dependencies = [ +# "pyyaml", +# ] +# /// + +""" +Script to filter nf-test paths based on ARM build failure markers. +This script checks for .arm-build-failure.yml files and filters out tests accordingly. +""" + +import json +import yaml +import os +import sys +from pathlib import Path + + +def has_arm_failure_marker(test_path): + """ + Check if a test path has an ARM failure marker in its module directory. + + Args: + test_path: Path to test file or module directory + + Returns: + tuple: (has_marker, module_dir, failure_info) + """ + # Extract module directory from test path + if test_path.startswith("modules/nf-core/"): + # For module tests like "modules/nf-core/fastqc" + module_dir = test_path + elif test_path.startswith("subworkflows/nf-core/"): + # For subworkflow tests like "subworkflows/nf-core/fastq_align_bwa" + module_dir = test_path + else: + # Try to extract module directory from path + parts = test_path.split("/") + if len(parts) >= 3 and parts[0] == "modules" and parts[1] == "nf-core": + module_dir = "/".join(parts[:3]) + elif len(parts) >= 3 and parts[0] == "subworkflows" and parts[1] == "nf-core": + module_dir = "/".join(parts[:3]) + else: + # Can't determine module directory + return False, None, None + + arm_failure_file = os.path.join(module_dir, ".arm-build-failure.yml") + + if os.path.exists(arm_failure_file): + try: + with open(arm_failure_file, 'r') as f: + failure_info = yaml.safe_load(f) or {} + return True, module_dir, failure_info + except Exception as e: + print(f"Warning: Could not read {arm_failure_file}: {e}", file=sys.stderr) + return True, module_dir, {} + + return False, module_dir, None + + +def filter_arm_failure_tests(test_paths): + """ + Filter test paths based on ARM failure markers. + + Args: + test_paths: List of test paths + + Returns: + List of filtered test paths (excluding those with ARM failures) + """ + filtered_paths = [] + + for test_path in test_paths: + has_failure, module_dir, failure_info = has_arm_failure_marker(test_path) + + if has_failure: + print(f"⚠️ Skipping tests for {test_path} (ARM build failure detected)", file=sys.stderr) + if failure_info: + print(f" Module: {module_dir}", file=sys.stderr) + print(f" Reason: {failure_info.get('reason', 'No reason specified')}", file=sys.stderr) + print(f" Date: {failure_info.get('date', 'No date specified')}", file=sys.stderr) + else: + filtered_paths.append(test_path) + + return filtered_paths + + +def main(): + """Main function to handle command line arguments and filter tests.""" + if len(sys.argv) != 2: + print("Usage: filter_arm_failure_tests.py ") + print("test_paths_json: JSON string of test paths") + sys.exit(1) + + test_paths_json = sys.argv[1] + + try: + test_paths = json.loads(test_paths_json) + except json.JSONDecodeError as e: + print(f"Error parsing test paths JSON: {e}") + sys.exit(1) + + if not test_paths: + # Return empty list if no paths + print(json.dumps([])) + return + + # Filter tests and output result + filtered_paths = filter_arm_failure_tests(test_paths) + + print(f"Original test count: {len(test_paths)}", file=sys.stderr) + print(f"Filtered test count: {len(filtered_paths)}", file=sys.stderr) + + if len(test_paths) != len(filtered_paths): + skipped_count = len(test_paths) - len(filtered_paths) + print(f"Skipped {skipped_count} test(s) due to ARM build failures", file=sys.stderr) + + # Output the filtered paths as JSON + print(json.dumps(filtered_paths)) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/manage_arm_failures.py b/.github/scripts/manage_arm_failures.py new file mode 100755 index 000000000000..ca063ce491e9 --- /dev/null +++ b/.github/scripts/manage_arm_failures.py @@ -0,0 +1,163 @@ +#!/usr/bin/env uv run +# /// script +# dependencies = [ +# "pyyaml", +# ] +# /// + +""" +Script to manage ARM build failure markers. +This script can add, remove, and list ARM failure markers for modules. +""" + +import yaml +import os +import sys +import glob +from datetime import datetime +from pathlib import Path + + +def find_modules(): + """Find all modules and subworkflows.""" + modules = glob.glob("modules/nf-core/*") + subworkflows = glob.glob("subworkflows/nf-core/*") + return sorted([m for m in modules + subworkflows if os.path.isdir(m)]) + + +def get_failure_marker_path(module_dir): + """Get the path to the ARM failure marker for a module.""" + return os.path.join(module_dir, ".arm-build-failure.yml") + + +def has_failure_marker(module_dir): + """Check if a module has an ARM failure marker.""" + return os.path.exists(get_failure_marker_path(module_dir)) + + +def create_failure_marker(module_dir, reason="Manual ARM build disable", reported_by="manual"): + """Create an ARM failure marker for a module.""" + failure_file = get_failure_marker_path(module_dir) + + failure_data = { + "reason": reason, + "date": datetime.now().strftime("%Y-%m-%d"), + "reported_by": reported_by + } + + with open(failure_file, 'w') as f: + f.write("# ARM Build Failure Marker\n") + f.write("# This file indicates that ARM builds are disabled for this module\n") + f.write("# Remove this file to re-enable ARM builds\n\n") + yaml.dump(failure_data, f, default_flow_style=False) + + print(f"✅ Created ARM failure marker for {module_dir}") + return failure_file + + +def remove_failure_marker(module_dir): + """Remove an ARM failure marker for a module.""" + failure_file = get_failure_marker_path(module_dir) + + if os.path.exists(failure_file): + os.remove(failure_file) + print(f"✅ Removed ARM failure marker for {module_dir}") + return True + else: + print(f"❌ No ARM failure marker found for {module_dir}") + return False + + +def list_failure_markers(): + """List all modules with ARM failure markers.""" + modules = find_modules() + failed_modules = [] + + for module_dir in modules: + if has_failure_marker(module_dir): + failure_file = get_failure_marker_path(module_dir) + try: + with open(failure_file, 'r') as f: + failure_info = yaml.safe_load(f) or {} + failed_modules.append((module_dir, failure_info)) + except Exception as e: + failed_modules.append((module_dir, {"error": str(e)})) + + if failed_modules: + print(f"📋 Found {len(failed_modules)} module(s) with ARM failure markers:") + print() + for module_dir, failure_info in failed_modules: + print(f"🔴 {module_dir}") + if "error" in failure_info: + print(f" Error reading marker: {failure_info['error']}") + else: + print(f" Reason: {failure_info.get('reason', 'No reason specified')}") + print(f" Date: {failure_info.get('date', 'No date specified')}") + print(f" Reported by: {failure_info.get('reported_by', 'Unknown')}") + print() + else: + print("✅ No modules with ARM failure markers found") + + return failed_modules + + +def main(): + """Main function to handle command line arguments.""" + if len(sys.argv) < 2: + print("Usage: manage_arm_failures.py [args]") + print("\nCommands:") + print(" list - List all modules with ARM failure markers") + print(" add [reason] - Add ARM failure marker to module") + print(" remove - Remove ARM failure marker from module") + print(" clean - Remove all ARM failure markers") + print("\nExamples:") + print(" manage_arm_failures.py list") + print(" manage_arm_failures.py add modules/nf-core/fastqc 'Known ARM issue'") + print(" manage_arm_failures.py remove modules/nf-core/fastqc") + sys.exit(1) + + command = sys.argv[1] + + if command == "list": + list_failure_markers() + + elif command == "add": + if len(sys.argv) < 3: + print("Error: Module directory required") + sys.exit(1) + + module_dir = sys.argv[2] + reason = sys.argv[3] if len(sys.argv) > 3 else "Manual ARM build disable" + + if not os.path.isdir(module_dir): + print(f"Error: Directory {module_dir} does not exist") + sys.exit(1) + + create_failure_marker(module_dir, reason, "manual") + + elif command == "remove": + if len(sys.argv) < 3: + print("Error: Module directory required") + sys.exit(1) + + module_dir = sys.argv[2] + remove_failure_marker(module_dir) + + elif command == "clean": + modules = find_modules() + removed_count = 0 + + for module_dir in modules: + if remove_failure_marker(module_dir): + removed_count += 1 + + print(f"✅ Removed {removed_count} ARM failure marker(s)") + + else: + print(f"Error: Unknown command '{command}'") + print("Use 'list', 'add', 'remove', or 'clean'") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/report_arm_build_failure.py b/.github/scripts/report_arm_build_failure.py new file mode 100755 index 000000000000..8859bc273c39 --- /dev/null +++ b/.github/scripts/report_arm_build_failure.py @@ -0,0 +1,120 @@ +#!/usr/bin/env uv run +# /// script +# dependencies = [ +# "pyyaml", +# ] +# /// + +""" +Script to filter ARM builds based on failure markers. +This script checks for .arm-build-failure.yml files and filters out ARM builds accordingly. +""" + +import json +import yaml +import os +import sys +from pathlib import Path + + +def filter_arm_builds(files_list, build_type="conda"): + """ + Filter ARM builds based on failure markers. + + Args: + files_list: List of file paths (environment.yml or Dockerfile) + build_type: Either "conda" or "dockerfile" + + Returns: + Dictionary with filtered matrix configuration + """ + filtered_combinations = [] + + for file_path in files_list: + # Extract module directory from file path + module_dir = os.path.dirname(file_path) + arm_failure_file = os.path.join(module_dir, ".arm-build-failure.yml") + + # Check if ARM failure marker exists + arm_build_disabled = os.path.exists(arm_failure_file) + + if arm_build_disabled: + print(f"⚠️ ARM builds disabled for {module_dir} (failure marker found)", file=sys.stderr) + # Read failure info for logging + try: + with open(arm_failure_file, 'r') as f: + failure_info = yaml.safe_load(f) + if failure_info: + print(f" Reason: {failure_info.get('reason', 'No reason specified')}", file=sys.stderr) + print(f" Date: {failure_info.get('date', 'No date specified')}", file=sys.stderr) + except Exception as e: + print(f" Could not read failure info: {e}", file=sys.stderr) + + # Add combinations based on build type + if build_type == "conda": + # Add all combinations for conda builds + for profile in ["docker", "singularity"]: + for platform in ["linux/amd64", "linux/arm64"]: + # Skip ARM builds if failure marker exists + if arm_build_disabled and platform == "linux/arm64": + print(f" Skipping {platform} build for {profile}", file=sys.stderr) + continue + + filtered_combinations.append({ + "files": file_path, + "profile": profile, + "platform": platform + }) + + elif build_type == "dockerfile": + # Add combinations for dockerfile builds + for platform in ["linux/amd64", "linux/arm64"]: + # Skip ARM builds if failure marker exists + if arm_build_disabled and platform == "linux/arm64": + print(f" Skipping {platform} Dockerfile build", file=sys.stderr) + continue + + filtered_combinations.append({ + "files": file_path, + "platform": platform + }) + + return {"include": filtered_combinations} + + +def main(): + """Main function to handle command line arguments and filter builds.""" + if len(sys.argv) != 3: + print("Usage: script.py ") + print("build_type: 'conda' or 'dockerfile'") + print("files_json: JSON string of file paths") + sys.exit(1) + + build_type = sys.argv[1] + files_json = sys.argv[2] + + if build_type not in ["conda", "dockerfile"]: + print("Error: build_type must be 'conda' or 'dockerfile'") + sys.exit(1) + + try: + files_list = json.loads(files_json) + except json.JSONDecodeError as e: + print(f"Error parsing files JSON: {e}") + sys.exit(1) + + if not files_list: + # Return empty matrix if no files + print(json.dumps({"include": []})) + return + + # Filter builds and output result + filtered_matrix = filter_arm_builds(files_list, build_type) + print(f"Filtered {build_type} matrix: {json.dumps(filtered_matrix, indent=2)}", file=sys.stderr) + + # Output just the JSON for GitHub Actions (to stdout) + print(json.dumps(filtered_matrix)) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 88b9be9d53e6..faea21c24bf3 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -110,7 +110,7 @@ jobs: NXF_ANSI_LOG: false TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} outputs: - filtered_paths: ${{ steps.filter.outputs.filtered_paths }} + arm_filtered_paths: ${{ steps.filter-arm.outputs.arm_filtered_paths }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -153,13 +153,24 @@ jobs: echo "filtered_paths=${FILTERED}" >> $GITHUB_OUTPUT + - name: Filter ARM failure tests + id: filter-arm + run: | + PATHS='${{ steps.filter.outputs.filtered_paths }}' + if [ "$PATHS" = "[]" ] || [ -z "$PATHS" ]; then + echo "arm_filtered_paths=[]" >> $GITHUB_OUTPUT + else + ARM_FILTERED=$(.github/scripts/filter_arm_failure_tests.py "$PATHS") + echo "arm_filtered_paths=${ARM_FILTERED}" >> $GITHUB_OUTPUT + fi + - name: debug run: | echo filtered_paths = ${{ steps.filter.outputs.filtered_paths }} + echo arm_filtered_paths = ${{ steps.filter-arm.outputs.arm_filtered_paths }} - name: Run nf-test Action - if: ${{steps.filter.outputs.filtered_paths != '[]'}} - continue-on-error: ${{ runner.arch == 'ARM64' }} + if: ${{steps.filter-arm.outputs.arm_filtered_paths != '[]'}} uses: ./.github/actions/nf-test-action env: SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} @@ -170,7 +181,7 @@ jobs: profile: ${{ matrix.profile }}${{ runner.arch == 'ARM64' && ',arm_CI,wave' || '' }} shard: ${{ matrix.shard }} total_shards: ${{ env.TOTAL_SHARDS }} - paths: "${{ join(fromJson(steps.filter.outputs.filtered_paths), ' ') }}" + paths: "${{ join(fromJson(steps.filter-arm.outputs.arm_filtered_paths), ' ') }}" confirm-pass-nf-test: runs-on: @@ -189,7 +200,7 @@ jobs: run: exit 1 - name: If no tests run, but filtered_paths was not empty and no tests succeeded, fail - if: ${{ needs.nf-test.outputs.filtered_paths != '' && !contains(needs.*.result, 'success') }} + if: ${{ needs.nf-test.outputs.arm_filtered_paths != '' && !contains(needs.*.result, 'success') }} run: exit 1 - name: All tests ok @@ -199,10 +210,10 @@ jobs: - name: debug-print if: always() run: | - echo "::group::DEBUG: `needs` Contents" + echo "::group::DEBUG: \`needs\` Contents" echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}" echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" - echo "DEBUG: needs.nf-test.outputs.filtered_paths = ${{ needs.nf-test.outputs.filtered_paths }}" + echo "DEBUG: needs.nf-test.outputs.arm_filtered_paths = ${{ needs.nf-test.outputs.arm_filtered_paths }}" echo "::endgroup::" - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 7297b30b1e92..18c48f262355 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -26,6 +26,8 @@ jobs: outputs: conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }} dockerfile-matrix: ${{ steps.docker-diff.outputs.all_changed_files }} + filtered-conda-matrix: ${{ steps.filter-conda.outputs.result }} + filtered-dockerfile-matrix: ${{ steps.filter-dockerfile.outputs.result }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -47,10 +49,26 @@ jobs: files: | modules/**/Dockerfile + - name: Filter conda builds for ARM failures + id: filter-conda + if: ${{ steps.conda-diff.outputs.all_changed_files != '[]' }} + run: | + # Use the external script to filter ARM builds + FILTERED_MATRIX=$(.github/scripts/report_arm_build_failure.py conda '${{ steps.conda-diff.outputs.all_changed_files }}' | tail -n 1) + echo "result=$FILTERED_MATRIX" >> $GITHUB_OUTPUT + + - name: Filter dockerfile builds for ARM failures + id: filter-dockerfile + if: ${{ steps.docker-diff.outputs.all_changed_files != '[]' }} + run: | + # Use the external script to filter ARM builds + FILTERED_MATRIX=$(.github/scripts/report_arm_build_failure.py dockerfile '${{ steps.docker-diff.outputs.all_changed_files }}' | tail -n 1) + echo "result=$FILTERED_MATRIX" >> $GITHUB_OUTPUT + conda-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }} + if: ${{ needs.generate-matrix.outputs.filtered-conda-matrix != '' && needs.generate-matrix.outputs.filtered-conda-matrix != '{"include":[]}' }} needs: generate-matrix name: Build ${{ matrix.profile }} ${{ matrix.platform }} Container runs-on: ubuntu-latest @@ -58,10 +76,8 @@ jobs: strategy: fail-fast: false max-parallel: 4 - matrix: - files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}" - profile: [docker, singularity] - platform: ["linux/amd64", "linux/arm64"] + matrix: ${{ fromJson(needs.generate-matrix.outputs.filtered-conda-matrix) }} + steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -73,7 +89,7 @@ jobs: - name: Build ${{ matrix.profile }} container # FIXME Hack while iron out the CI - continue-on-error: true + continue-on-error: false env: PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }} run: | @@ -85,10 +101,32 @@ jobs: --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} + - name: Mark ARM build failure + if: ${{ failure() && matrix.platform == 'linux/arm64' }} + run: | + MODULE_DIR=$(dirname "${{ matrix.files }}") + FAILURE_FILE="$MODULE_DIR/.arm-build-failure.yml" + + # Create failure marker + cat > "$FAILURE_FILE" << EOF + # ARM Build Failure Marker + # This file indicates that ARM builds are disabled for this module + # Remove this file to re-enable ARM builds + + reason: Wave ${{ matrix.profile }} build failed on ARM platform + date: '$(date -I)' + reported_by: github-actions + workflow_run: ${{ github.run_id }} + commit_sha: ${{ github.sha }} + EOF + + echo "Created ARM failure marker at $FAILURE_FILE" + cat "$FAILURE_FILE" + dockerfile-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: ${{ needs.generate-matrix.outputs.dockerfile-matrix != '[]' }} + if: ${{ needs.generate-matrix.outputs.filtered-dockerfile-matrix != '' && needs.generate-matrix.outputs.filtered-dockerfile-matrix != '{"include":[]}' }} needs: generate-matrix name: Build Dockerfile-based ${{ matrix.platform }} Container runs-on: ubuntu-latest @@ -96,12 +134,7 @@ jobs: strategy: fail-fast: false max-parallel: 4 - matrix: - files: "${{ fromJson(needs.generate-matrix.outputs.dockerfile-matrix) }}" - # NOTE singularity build requires a Singularity definition file in place of Dockerfile - # https://nextflow.slack.com/archives/C0477AS31T5/p1731755350230149?thread_ts=1731697852.661849&cid=C0477AS31T5 - # profile: [docker] - platform: ["linux/amd64", "linux/arm64"] + matrix: ${{ fromJson(needs.generate-matrix.outputs.filtered-dockerfile-matrix) }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -132,6 +165,28 @@ jobs: --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} + - name: Mark ARM build failure + if: ${{ failure() && matrix.platform == 'linux/arm64' }} + run: | + MODULE_DIR=$(dirname "${{ matrix.files }}") + FAILURE_FILE="$MODULE_DIR/.arm-build-failure.yml" + + # Create failure marker + cat > "$FAILURE_FILE" << EOF + # ARM Build Failure Marker + # This file indicates that ARM builds are disabled for this module + # Remove this file to re-enable ARM builds + + reason: Wave Dockerfile build failed on ARM platform + date: '$(date -I)' + reported_by: github-actions + workflow_run: ${{ github.run_id }} + commit_sha: ${{ github.sha }} + EOF + + echo "Created ARM failure marker at $FAILURE_FILE" + cat "$FAILURE_FILE" + # bump-versions: # needs: generate-matrix # name: bump-versions diff --git a/docs/ARM_BUILD_FAILURES.md b/docs/ARM_BUILD_FAILURES.md new file mode 100644 index 000000000000..5d47d539975d --- /dev/null +++ b/docs/ARM_BUILD_FAILURES.md @@ -0,0 +1,200 @@ +# ARM Build Failure Management + +This document describes the system for managing ARM build failures in nf-core modules. + +## Overview + +Some modules may fail to build on ARM64 architecture due to various reasons such as: +- Dependencies not available for ARM64 +- Architecture-specific compilation issues +- Upstream package limitations + +To handle these cases gracefully, we have implemented a system that allows temporarily disabling ARM builds for specific modules until the issues are resolved. + +## How It Works + +### Failure Marker Files + +When a module consistently fails ARM builds, you can create a `.arm-build-failure.yml` file in the module directory. This file: + +1. **Disables ARM builds** for that module in CI/CD pipelines +2. **Documents the failure reason** and relevant information +3. **Can be easily removed** when the issue is resolved + +### Workflow Integration + +The GitHub Actions workflow automatically: +- Checks for `.arm-build-failure.yml` files before building containers +- Skips ARM builds (`linux/arm64`) for modules with failure markers +- Continues with AMD64 builds (`linux/amd64`) as normal +- Logs the reason for skipping ARM builds + +## Managing ARM Build Failures + +### Using the Helper Script + +We provide a helper script to manage ARM build failure markers: + +```bash +# Add a failure marker +./scripts/manage-arm-failures.py add fastqc \ + --reason "Package xyz not available for ARM64" \ + --reported-by "your-github-username" \ + --issue-url "https://github.com/nf-core/modules/issues/12345" + +# Check if a module has ARM builds disabled +./scripts/manage-arm-failures.py check fastqc + +# List all modules with ARM build failures +./scripts/manage-arm-failures.py list + +# Remove a failure marker (re-enable ARM builds) +./scripts/manage-arm-failures.py remove fastqc +``` + +### Manual Management + +You can also manually create/edit the `.arm-build-failure.yml` files: + +```yaml +# ARM Build Failure Marker +# This file indicates that ARM builds are disabled for this module +# Remove this file to re-enable ARM builds + +reason: "Package XYZ not available for ARM64 architecture" +date: "2024-01-15" +reported_by: "github-username" +issue_url: "https://github.com/nf-core/modules/issues/12345" +error_details: | + The conda package 'some-package' does not have ARM64 builds available. + This causes the wave container build to fail consistently. +notes: | + Alternative solutions investigated: + - Tried using different base image: failed + - Contacted upstream maintainer: waiting for response +``` + +## File Format + +The `.arm-build-failure.yml` file supports the following fields: + +- **`reason`** (required): Brief description of why ARM builds fail +- **`date`** (required): Date when the failure was first reported +- **`reported_by`** (optional): GitHub username of the person reporting the issue +- **`issue_url`** (optional): Link to the related GitHub issue +- **`error_details`** (optional): Detailed error information or logs +- **`notes`** (optional): Additional notes, investigation details, or workarounds attempted + +## Workflow Impact + +### Conda-based Builds (`conda-wave` job) + +- Modules with `.arm-build-failure.yml` will skip ARM builds +- Docker and Singularity profiles are both affected +- AMD64 builds continue normally + +### Dockerfile-based Builds (`dockerfile-wave` job) + +- Modules with `.arm-build-failure.yml` will skip ARM builds +- AMD64 builds continue normally + +### CI/CD Logs + +When ARM builds are skipped, you'll see logs like: + +``` +⚠️ ARM builds disabled for modules/nf-core/fastqc (failure marker found) + Reason: Package XYZ not available for ARM64 architecture + Date: 2024-01-15 + Skipping linux/arm64 build for docker + Skipping linux/arm64 build for singularity +``` + +## Best Practices + +### When to Add Failure Markers + +- ARM builds consistently fail for legitimate architecture reasons +- The failure is not due to temporary issues (network, quota, etc.) +- The issue has been investigated and documented +- An upstream issue has been reported (when applicable) + +### Documentation Requirements + +When adding a failure marker, please include: +- A clear reason for the failure +- Link to any related GitHub issues +- Details of investigation or workarounds attempted +- Date when the failure was first observed + +### Regular Review + +Periodically review ARM build failures: +- Check if upstream issues have been resolved +- Test if packages have become available for ARM64 +- Remove markers when issues are fixed + +### Re-enabling ARM Builds + +To re-enable ARM builds for a module: +1. Remove the `.arm-build-failure.yml` file +2. Test the ARM build manually (if possible) +3. Monitor the CI for successful ARM builds + +## Examples + +### Example 1: Package Not Available + +```yaml +reason: "bioconda package 'tool-xyz' has no ARM64 builds" +date: "2024-01-15" +reported_by: "maintainer-username" +issue_url: "https://github.com/bioconda/bioconda-recipes/issues/12345" +error_details: | + Wave build fails with: + PackagesNotFoundError: The following packages are not available from current channels: + - tool-xyz[version='>=1.0.0',build=*linux-aarch64] +notes: | + Contacted bioconda maintainers about ARM64 support. + Alternative tools investigated but none suitable. +``` + +### Example 2: Architecture-Specific Compilation Issue + +```yaml +reason: "Compilation fails on ARM64 due to assembly code" +date: "2024-02-01" +reported_by: "developer-username" +issue_url: "https://github.com/upstream/tool/issues/456" +error_details: | + Build fails during compilation with: + Error: unsupported instruction on ARM64 architecture +notes: | + Upstream aware of issue, fix planned for next major release. + No workaround available currently. +``` + +## Troubleshooting + +### Script Not Working + +If the management script doesn't work: +1. Ensure you're in the repository root +2. Check that `uv` is installed +3. Verify the modules directory structure + +### Workflow Still Running ARM Builds + +If ARM builds still run despite having a failure marker: +1. Check the file name is exactly `.arm-build-failure.yml` +2. Verify the file is in the correct module directory +3. Ensure the YAML syntax is valid +4. Check the workflow logs for any errors in the filtering step + +### Re-enabling Builds + +To test if ARM builds can be re-enabled: +1. Remove the failure marker file +2. Create a small PR that touches the module's `environment.yml` +3. Monitor the CI workflow for successful ARM builds +4. If still failing, re-add the marker with updated information diff --git a/modules/nf-core/fastqc/.arm-build-failure.yml b/modules/nf-core/fastqc/.arm-build-failure.yml new file mode 100644 index 000000000000..d2dfab265120 --- /dev/null +++ b/modules/nf-core/fastqc/.arm-build-failure.yml @@ -0,0 +1,7 @@ +# ARM Build Failure Marker +# This file indicates that ARM builds are disabled for this module +# Remove this file to re-enable ARM builds + +reason: Testing ARM failure filtering +date: '2025-06-02' +reported_by: test-user diff --git a/modules/nf-core/kraken2/build/tests/main.nf.test b/modules/nf-core/kraken2/build/tests/main.nf.test index d3d26bcd71fe..fbeb91a827e3 100644 --- a/modules/nf-core/kraken2/build/tests/main.nf.test +++ b/modules/nf-core/kraken2/build/tests/main.nf.test @@ -128,6 +128,8 @@ nextflow_process { test("sarscov2 protein_db stub") { + tag "arm_failure" + options "-stub" setup { @@ -170,4 +172,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/scripts/manage-arm-failures.py b/scripts/manage-arm-failures.py new file mode 100644 index 000000000000..8a3ac5a532ab --- /dev/null +++ b/scripts/manage-arm-failures.py @@ -0,0 +1,176 @@ +#!/usr/bin/env uv run +# /// script +# dependencies = [ +# "pyyaml", +# "click", +# ] +# /// + +""" +Helper script to manage ARM build failure markers for nf-core modules. + +This script allows you to: +- Add ARM build failure markers to modules +- Remove ARM build failure markers from modules +- List all modules with ARM build failures +- Check if a specific module has ARM build failures +""" + +import click +import yaml +import os +from pathlib import Path +from datetime import datetime + + +FAILURE_MARKER_FILENAME = ".arm-build-failure.yml" + + +def get_modules_dir(): + """Get the modules directory path.""" + script_dir = Path(__file__).parent + modules_dir = script_dir.parent / "modules" / "nf-core" + if not modules_dir.exists(): + raise click.ClickException(f"Modules directory not found: {modules_dir}") + return modules_dir + + +def get_module_path(module_name): + """Get the path to a specific module.""" + modules_dir = get_modules_dir() + module_path = modules_dir / module_name + if not module_path.exists(): + raise click.ClickException(f"Module not found: {module_name}") + return module_path + + +def get_failure_marker_path(module_name): + """Get the path to the ARM failure marker file for a module.""" + module_path = get_module_path(module_name) + return module_path / FAILURE_MARKER_FILENAME + + +@click.group() +def cli(): + """Manage ARM build failure markers for nf-core modules.""" + pass + + +@cli.command() +@click.argument('module_name') +@click.option('--reason', '-r', required=True, help='Reason for ARM build failure') +@click.option('--reported-by', '-u', help='GitHub username of reporter') +@click.option('--issue-url', '-i', help='URL to related GitHub issue') +@click.option('--error-details', '-e', help='Detailed error information') +@click.option('--notes', '-n', help='Additional notes') +def add(module_name, reason, reported_by, issue_url, error_details, notes): + """Add an ARM build failure marker to a module.""" + failure_marker_path = get_failure_marker_path(module_name) + + if failure_marker_path.exists(): + if not click.confirm(f"ARM failure marker already exists for {module_name}. Overwrite?"): + return + + failure_data = { + 'reason': reason, + 'date': datetime.now().strftime('%Y-%m-%d'), + } + + if reported_by: + failure_data['reported_by'] = reported_by + if issue_url: + failure_data['issue_url'] = issue_url + if error_details: + failure_data['error_details'] = error_details + if notes: + failure_data['notes'] = notes + + with open(failure_marker_path, 'w') as f: + f.write("# ARM Build Failure Marker\n") + f.write("# This file indicates that ARM builds are disabled for this module\n") + f.write("# Remove this file to re-enable ARM builds\n\n") + yaml.dump(failure_data, f, default_flow_style=False, sort_keys=False) + + click.echo(f"✅ Added ARM failure marker for {module_name}") + + +@cli.command() +@click.argument('module_name') +def remove(module_name): + """Remove an ARM build failure marker from a module.""" + failure_marker_path = get_failure_marker_path(module_name) + + if not failure_marker_path.exists(): + click.echo(f"❌ No ARM failure marker found for {module_name}") + return + + if click.confirm(f"Remove ARM failure marker for {module_name}?"): + failure_marker_path.unlink() + click.echo(f"✅ Removed ARM failure marker for {module_name}") + + +@cli.command() +@click.argument('module_name') +def check(module_name): + """Check if a module has an ARM build failure marker.""" + failure_marker_path = get_failure_marker_path(module_name) + + if failure_marker_path.exists(): + click.echo(f"⚠️ {module_name} has ARM builds disabled") + + try: + with open(failure_marker_path, 'r') as f: + failure_data = yaml.safe_load(f) + if failure_data: + click.echo(f" Reason: {failure_data.get('reason', 'Not specified')}") + click.echo(f" Date: {failure_data.get('date', 'Not specified')}") + if 'reported_by' in failure_data: + click.echo(f" Reported by: {failure_data['reported_by']}") + if 'issue_url' in failure_data: + click.echo(f" Issue: {failure_data['issue_url']}") + except Exception as e: + click.echo(f" Error reading failure data: {e}") + else: + click.echo(f"✅ {module_name} has ARM builds enabled") + + +@cli.command() +def list(): + """List all modules with ARM build failure markers.""" + modules_dir = get_modules_dir() + failed_modules = [] + + for module_dir in modules_dir.iterdir(): + if module_dir.is_dir(): + failure_marker = module_dir / FAILURE_MARKER_FILENAME + if failure_marker.exists(): + try: + with open(failure_marker, 'r') as f: + failure_data = yaml.safe_load(f) + failed_modules.append({ + 'name': module_dir.name, + 'reason': failure_data.get('reason', 'Not specified') if failure_data else 'Not specified', + 'date': failure_data.get('date', 'Not specified') if failure_data else 'Not specified' + }) + except Exception as e: + failed_modules.append({ + 'name': module_dir.name, + 'reason': f'Error reading file: {e}', + 'date': 'Unknown' + }) + + if not failed_modules: + click.echo("✅ No modules have ARM build failures") + return + + click.echo(f"⚠️ Found {len(failed_modules)} modules with ARM build failures:\n") + + for module in sorted(failed_modules, key=lambda x: x['name']): + click.echo(f" {module['name']}") + click.echo(f" Reason: {module['reason']}") + click.echo(f" Date: {module['date']}") + click.echo() + + +if __name__ == '__main__': + cli() From 38174a0464a52dd4527fb016a30830faf000d263 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 2 Jun 2025 12:35:16 -0500 Subject: [PATCH 03/14] ci: Replace ARM failure management with skip marker system - Removed scripts for managing ARM build failures and filtering tests. - Introduced `filter_arm_tests.py` to skip ARM tests based on `.skip-arm` marker files. - Updated GitHub Actions workflows to utilize the new skip marker system. - Revised documentation to reflect changes in ARM test management. --- .github/scripts/filter_arm_failure_tests.py | 124 ----------- .github/scripts/filter_arm_tests.py | 25 +++ .github/scripts/manage_arm_failures.py | 163 --------------- .github/scripts/report_arm_build_failure.py | 120 ----------- .github/workflows/nf-test.yml | 10 +- .github/workflows/wave.yml | 60 ++---- docs/ARM_BUILD_FAILURES.md | 221 +++++++++----------- scripts/manage-arm-failures.py | 176 ---------------- 8 files changed, 136 insertions(+), 763 deletions(-) delete mode 100755 .github/scripts/filter_arm_failure_tests.py create mode 100755 .github/scripts/filter_arm_tests.py delete mode 100755 .github/scripts/manage_arm_failures.py delete mode 100755 .github/scripts/report_arm_build_failure.py delete mode 100644 scripts/manage-arm-failures.py diff --git a/.github/scripts/filter_arm_failure_tests.py b/.github/scripts/filter_arm_failure_tests.py deleted file mode 100755 index e3682767700a..000000000000 --- a/.github/scripts/filter_arm_failure_tests.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env uv run -# /// script -# dependencies = [ -# "pyyaml", -# ] -# /// - -""" -Script to filter nf-test paths based on ARM build failure markers. -This script checks for .arm-build-failure.yml files and filters out tests accordingly. -""" - -import json -import yaml -import os -import sys -from pathlib import Path - - -def has_arm_failure_marker(test_path): - """ - Check if a test path has an ARM failure marker in its module directory. - - Args: - test_path: Path to test file or module directory - - Returns: - tuple: (has_marker, module_dir, failure_info) - """ - # Extract module directory from test path - if test_path.startswith("modules/nf-core/"): - # For module tests like "modules/nf-core/fastqc" - module_dir = test_path - elif test_path.startswith("subworkflows/nf-core/"): - # For subworkflow tests like "subworkflows/nf-core/fastq_align_bwa" - module_dir = test_path - else: - # Try to extract module directory from path - parts = test_path.split("/") - if len(parts) >= 3 and parts[0] == "modules" and parts[1] == "nf-core": - module_dir = "/".join(parts[:3]) - elif len(parts) >= 3 and parts[0] == "subworkflows" and parts[1] == "nf-core": - module_dir = "/".join(parts[:3]) - else: - # Can't determine module directory - return False, None, None - - arm_failure_file = os.path.join(module_dir, ".arm-build-failure.yml") - - if os.path.exists(arm_failure_file): - try: - with open(arm_failure_file, 'r') as f: - failure_info = yaml.safe_load(f) or {} - return True, module_dir, failure_info - except Exception as e: - print(f"Warning: Could not read {arm_failure_file}: {e}", file=sys.stderr) - return True, module_dir, {} - - return False, module_dir, None - - -def filter_arm_failure_tests(test_paths): - """ - Filter test paths based on ARM failure markers. - - Args: - test_paths: List of test paths - - Returns: - List of filtered test paths (excluding those with ARM failures) - """ - filtered_paths = [] - - for test_path in test_paths: - has_failure, module_dir, failure_info = has_arm_failure_marker(test_path) - - if has_failure: - print(f"⚠️ Skipping tests for {test_path} (ARM build failure detected)", file=sys.stderr) - if failure_info: - print(f" Module: {module_dir}", file=sys.stderr) - print(f" Reason: {failure_info.get('reason', 'No reason specified')}", file=sys.stderr) - print(f" Date: {failure_info.get('date', 'No date specified')}", file=sys.stderr) - else: - filtered_paths.append(test_path) - - return filtered_paths - - -def main(): - """Main function to handle command line arguments and filter tests.""" - if len(sys.argv) != 2: - print("Usage: filter_arm_failure_tests.py ") - print("test_paths_json: JSON string of test paths") - sys.exit(1) - - test_paths_json = sys.argv[1] - - try: - test_paths = json.loads(test_paths_json) - except json.JSONDecodeError as e: - print(f"Error parsing test paths JSON: {e}") - sys.exit(1) - - if not test_paths: - # Return empty list if no paths - print(json.dumps([])) - return - - # Filter tests and output result - filtered_paths = filter_arm_failure_tests(test_paths) - - print(f"Original test count: {len(test_paths)}", file=sys.stderr) - print(f"Filtered test count: {len(filtered_paths)}", file=sys.stderr) - - if len(test_paths) != len(filtered_paths): - skipped_count = len(test_paths) - len(filtered_paths) - print(f"Skipped {skipped_count} test(s) due to ARM build failures", file=sys.stderr) - - # Output the filtered paths as JSON - print(json.dumps(filtered_paths)) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/filter_arm_tests.py b/.github/scripts/filter_arm_tests.py new file mode 100755 index 000000000000..09504ed1bfc9 --- /dev/null +++ b/.github/scripts/filter_arm_tests.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +"""Skip ARM tests for modules with .skip-arm marker files""" +import json, sys, os + +def main(): + if len(sys.argv) != 2: + print(json.dumps([])) + return + + paths = json.loads(sys.argv[1]) + filtered = [] + + for path in paths: + # Extract module dir (modules/nf-core/xxx or subworkflows/nf-core/xxx) + parts = path.split("/") + if len(parts) >= 3 and parts[1] == "nf-core": + module_dir = "/".join(parts[:3]) + if os.path.exists(f"{module_dir}/.skip-arm"): + print(f"⚠️ Skipping ARM tests for {path}", file=sys.stderr) + continue + filtered.append(path) + + print(json.dumps(filtered)) + +if __name__ == "__main__": main() diff --git a/.github/scripts/manage_arm_failures.py b/.github/scripts/manage_arm_failures.py deleted file mode 100755 index ca063ce491e9..000000000000 --- a/.github/scripts/manage_arm_failures.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env uv run -# /// script -# dependencies = [ -# "pyyaml", -# ] -# /// - -""" -Script to manage ARM build failure markers. -This script can add, remove, and list ARM failure markers for modules. -""" - -import yaml -import os -import sys -import glob -from datetime import datetime -from pathlib import Path - - -def find_modules(): - """Find all modules and subworkflows.""" - modules = glob.glob("modules/nf-core/*") - subworkflows = glob.glob("subworkflows/nf-core/*") - return sorted([m for m in modules + subworkflows if os.path.isdir(m)]) - - -def get_failure_marker_path(module_dir): - """Get the path to the ARM failure marker for a module.""" - return os.path.join(module_dir, ".arm-build-failure.yml") - - -def has_failure_marker(module_dir): - """Check if a module has an ARM failure marker.""" - return os.path.exists(get_failure_marker_path(module_dir)) - - -def create_failure_marker(module_dir, reason="Manual ARM build disable", reported_by="manual"): - """Create an ARM failure marker for a module.""" - failure_file = get_failure_marker_path(module_dir) - - failure_data = { - "reason": reason, - "date": datetime.now().strftime("%Y-%m-%d"), - "reported_by": reported_by - } - - with open(failure_file, 'w') as f: - f.write("# ARM Build Failure Marker\n") - f.write("# This file indicates that ARM builds are disabled for this module\n") - f.write("# Remove this file to re-enable ARM builds\n\n") - yaml.dump(failure_data, f, default_flow_style=False) - - print(f"✅ Created ARM failure marker for {module_dir}") - return failure_file - - -def remove_failure_marker(module_dir): - """Remove an ARM failure marker for a module.""" - failure_file = get_failure_marker_path(module_dir) - - if os.path.exists(failure_file): - os.remove(failure_file) - print(f"✅ Removed ARM failure marker for {module_dir}") - return True - else: - print(f"❌ No ARM failure marker found for {module_dir}") - return False - - -def list_failure_markers(): - """List all modules with ARM failure markers.""" - modules = find_modules() - failed_modules = [] - - for module_dir in modules: - if has_failure_marker(module_dir): - failure_file = get_failure_marker_path(module_dir) - try: - with open(failure_file, 'r') as f: - failure_info = yaml.safe_load(f) or {} - failed_modules.append((module_dir, failure_info)) - except Exception as e: - failed_modules.append((module_dir, {"error": str(e)})) - - if failed_modules: - print(f"📋 Found {len(failed_modules)} module(s) with ARM failure markers:") - print() - for module_dir, failure_info in failed_modules: - print(f"🔴 {module_dir}") - if "error" in failure_info: - print(f" Error reading marker: {failure_info['error']}") - else: - print(f" Reason: {failure_info.get('reason', 'No reason specified')}") - print(f" Date: {failure_info.get('date', 'No date specified')}") - print(f" Reported by: {failure_info.get('reported_by', 'Unknown')}") - print() - else: - print("✅ No modules with ARM failure markers found") - - return failed_modules - - -def main(): - """Main function to handle command line arguments.""" - if len(sys.argv) < 2: - print("Usage: manage_arm_failures.py [args]") - print("\nCommands:") - print(" list - List all modules with ARM failure markers") - print(" add [reason] - Add ARM failure marker to module") - print(" remove - Remove ARM failure marker from module") - print(" clean - Remove all ARM failure markers") - print("\nExamples:") - print(" manage_arm_failures.py list") - print(" manage_arm_failures.py add modules/nf-core/fastqc 'Known ARM issue'") - print(" manage_arm_failures.py remove modules/nf-core/fastqc") - sys.exit(1) - - command = sys.argv[1] - - if command == "list": - list_failure_markers() - - elif command == "add": - if len(sys.argv) < 3: - print("Error: Module directory required") - sys.exit(1) - - module_dir = sys.argv[2] - reason = sys.argv[3] if len(sys.argv) > 3 else "Manual ARM build disable" - - if not os.path.isdir(module_dir): - print(f"Error: Directory {module_dir} does not exist") - sys.exit(1) - - create_failure_marker(module_dir, reason, "manual") - - elif command == "remove": - if len(sys.argv) < 3: - print("Error: Module directory required") - sys.exit(1) - - module_dir = sys.argv[2] - remove_failure_marker(module_dir) - - elif command == "clean": - modules = find_modules() - removed_count = 0 - - for module_dir in modules: - if remove_failure_marker(module_dir): - removed_count += 1 - - print(f"✅ Removed {removed_count} ARM failure marker(s)") - - else: - print(f"Error: Unknown command '{command}'") - print("Use 'list', 'add', 'remove', or 'clean'") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/report_arm_build_failure.py b/.github/scripts/report_arm_build_failure.py deleted file mode 100755 index 8859bc273c39..000000000000 --- a/.github/scripts/report_arm_build_failure.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env uv run -# /// script -# dependencies = [ -# "pyyaml", -# ] -# /// - -""" -Script to filter ARM builds based on failure markers. -This script checks for .arm-build-failure.yml files and filters out ARM builds accordingly. -""" - -import json -import yaml -import os -import sys -from pathlib import Path - - -def filter_arm_builds(files_list, build_type="conda"): - """ - Filter ARM builds based on failure markers. - - Args: - files_list: List of file paths (environment.yml or Dockerfile) - build_type: Either "conda" or "dockerfile" - - Returns: - Dictionary with filtered matrix configuration - """ - filtered_combinations = [] - - for file_path in files_list: - # Extract module directory from file path - module_dir = os.path.dirname(file_path) - arm_failure_file = os.path.join(module_dir, ".arm-build-failure.yml") - - # Check if ARM failure marker exists - arm_build_disabled = os.path.exists(arm_failure_file) - - if arm_build_disabled: - print(f"⚠️ ARM builds disabled for {module_dir} (failure marker found)", file=sys.stderr) - # Read failure info for logging - try: - with open(arm_failure_file, 'r') as f: - failure_info = yaml.safe_load(f) - if failure_info: - print(f" Reason: {failure_info.get('reason', 'No reason specified')}", file=sys.stderr) - print(f" Date: {failure_info.get('date', 'No date specified')}", file=sys.stderr) - except Exception as e: - print(f" Could not read failure info: {e}", file=sys.stderr) - - # Add combinations based on build type - if build_type == "conda": - # Add all combinations for conda builds - for profile in ["docker", "singularity"]: - for platform in ["linux/amd64", "linux/arm64"]: - # Skip ARM builds if failure marker exists - if arm_build_disabled and platform == "linux/arm64": - print(f" Skipping {platform} build for {profile}", file=sys.stderr) - continue - - filtered_combinations.append({ - "files": file_path, - "profile": profile, - "platform": platform - }) - - elif build_type == "dockerfile": - # Add combinations for dockerfile builds - for platform in ["linux/amd64", "linux/arm64"]: - # Skip ARM builds if failure marker exists - if arm_build_disabled and platform == "linux/arm64": - print(f" Skipping {platform} Dockerfile build", file=sys.stderr) - continue - - filtered_combinations.append({ - "files": file_path, - "platform": platform - }) - - return {"include": filtered_combinations} - - -def main(): - """Main function to handle command line arguments and filter builds.""" - if len(sys.argv) != 3: - print("Usage: script.py ") - print("build_type: 'conda' or 'dockerfile'") - print("files_json: JSON string of file paths") - sys.exit(1) - - build_type = sys.argv[1] - files_json = sys.argv[2] - - if build_type not in ["conda", "dockerfile"]: - print("Error: build_type must be 'conda' or 'dockerfile'") - sys.exit(1) - - try: - files_list = json.loads(files_json) - except json.JSONDecodeError as e: - print(f"Error parsing files JSON: {e}") - sys.exit(1) - - if not files_list: - # Return empty matrix if no files - print(json.dumps({"include": []})) - return - - # Filter builds and output result - filtered_matrix = filter_arm_builds(files_list, build_type) - print(f"Filtered {build_type} matrix: {json.dumps(filtered_matrix, indent=2)}", file=sys.stderr) - - # Output just the JSON for GitHub Actions (to stdout) - print(json.dumps(filtered_matrix)) - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index faea21c24bf3..27708101a5f4 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -153,16 +153,12 @@ jobs: echo "filtered_paths=${FILTERED}" >> $GITHUB_OUTPUT - - name: Filter ARM failure tests + - name: Filter ARM tests id: filter-arm run: | PATHS='${{ steps.filter.outputs.filtered_paths }}' - if [ "$PATHS" = "[]" ] || [ -z "$PATHS" ]; then - echo "arm_filtered_paths=[]" >> $GITHUB_OUTPUT - else - ARM_FILTERED=$(.github/scripts/filter_arm_failure_tests.py "$PATHS") - echo "arm_filtered_paths=${ARM_FILTERED}" >> $GITHUB_OUTPUT - fi + ARM_FILTERED=$(.github/scripts/filter_arm_tests.py "$PATHS") + echo "arm_filtered_paths=${ARM_FILTERED}" >> $GITHUB_OUTPUT - name: debug run: | diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 18c48f262355..eba3a7f6a9a0 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -49,22 +49,6 @@ jobs: files: | modules/**/Dockerfile - - name: Filter conda builds for ARM failures - id: filter-conda - if: ${{ steps.conda-diff.outputs.all_changed_files != '[]' }} - run: | - # Use the external script to filter ARM builds - FILTERED_MATRIX=$(.github/scripts/report_arm_build_failure.py conda '${{ steps.conda-diff.outputs.all_changed_files }}' | tail -n 1) - echo "result=$FILTERED_MATRIX" >> $GITHUB_OUTPUT - - - name: Filter dockerfile builds for ARM failures - id: filter-dockerfile - if: ${{ steps.docker-diff.outputs.all_changed_files != '[]' }} - run: | - # Use the external script to filter ARM builds - FILTERED_MATRIX=$(.github/scripts/report_arm_build_failure.py dockerfile '${{ steps.docker-diff.outputs.all_changed_files }}' | tail -n 1) - echo "result=$FILTERED_MATRIX" >> $GITHUB_OUTPUT - conda-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' @@ -105,23 +89,13 @@ jobs: if: ${{ failure() && matrix.platform == 'linux/arm64' }} run: | MODULE_DIR=$(dirname "${{ matrix.files }}") - FAILURE_FILE="$MODULE_DIR/.arm-build-failure.yml" + SKIP_FILE="$MODULE_DIR/.skip-arm" - # Create failure marker - cat > "$FAILURE_FILE" << EOF - # ARM Build Failure Marker - # This file indicates that ARM builds are disabled for this module - # Remove this file to re-enable ARM builds + # Create simple skip marker with workflow run link + echo "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" > "$SKIP_FILE" - reason: Wave ${{ matrix.profile }} build failed on ARM platform - date: '$(date -I)' - reported_by: github-actions - workflow_run: ${{ github.run_id }} - commit_sha: ${{ github.sha }} - EOF - - echo "Created ARM failure marker at $FAILURE_FILE" - cat "$FAILURE_FILE" + echo "Created ARM skip marker at $SKIP_FILE" + cat "$SKIP_FILE" dockerfile-wave: # NOTE This should get skipped because generate-matrix won't run @@ -169,23 +143,13 @@ jobs: if: ${{ failure() && matrix.platform == 'linux/arm64' }} run: | MODULE_DIR=$(dirname "${{ matrix.files }}") - FAILURE_FILE="$MODULE_DIR/.arm-build-failure.yml" - - # Create failure marker - cat > "$FAILURE_FILE" << EOF - # ARM Build Failure Marker - # This file indicates that ARM builds are disabled for this module - # Remove this file to re-enable ARM builds - - reason: Wave Dockerfile build failed on ARM platform - date: '$(date -I)' - reported_by: github-actions - workflow_run: ${{ github.run_id }} - commit_sha: ${{ github.sha }} - EOF - - echo "Created ARM failure marker at $FAILURE_FILE" - cat "$FAILURE_FILE" + SKIP_FILE="$MODULE_DIR/.skip-arm" + + # Create simple skip marker with workflow run link + echo "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" > "$SKIP_FILE" + + echo "Created ARM skip marker at $SKIP_FILE" + cat "$SKIP_FILE" # bump-versions: # needs: generate-matrix diff --git a/docs/ARM_BUILD_FAILURES.md b/docs/ARM_BUILD_FAILURES.md index 5d47d539975d..4b87a2e1ccb0 100644 --- a/docs/ARM_BUILD_FAILURES.md +++ b/docs/ARM_BUILD_FAILURES.md @@ -1,200 +1,171 @@ -# ARM Build Failure Management +# ARM Test Skipping -This document describes the system for managing ARM build failures in nf-core modules. +This document describes the simple system for skipping ARM tests in nf-core modules. ## Overview -Some modules may fail to build on ARM64 architecture due to various reasons such as: +Some modules may fail to run tests on ARM64 architecture due to various reasons such as: - Dependencies not available for ARM64 - Architecture-specific compilation issues - Upstream package limitations -To handle these cases gracefully, we have implemented a system that allows temporarily disabling ARM builds for specific modules until the issues are resolved. +To handle these cases gracefully, we have implemented a simple system that allows temporarily disabling ARM tests for specific modules until the issues are resolved. ## How It Works -### Failure Marker Files +### Skip Marker Files -When a module consistently fails ARM builds, you can create a `.arm-build-failure.yml` file in the module directory. This file: +When a module consistently fails ARM tests, you can create a `.skip-arm` file in the module directory. This file: -1. **Disables ARM builds** for that module in CI/CD pipelines -2. **Documents the failure reason** and relevant information +1. **Disables ARM tests** for that module in CI/CD pipelines +2. **Can contain a link** to the wave build failure (optional) 3. **Can be easily removed** when the issue is resolved ### Workflow Integration The GitHub Actions workflow automatically: -- Checks for `.arm-build-failure.yml` files before building containers -- Skips ARM builds (`linux/arm64`) for modules with failure markers -- Continues with AMD64 builds (`linux/amd64`) as normal -- Logs the reason for skipping ARM builds +- Checks for `.skip-arm` files before running tests on ARM runners +- Skips ARM tests for modules with skip markers +- Continues with AMD64 tests as normal +- Logs which tests are being skipped -## Managing ARM Build Failures +## Managing ARM Test Skipping -### Using the Helper Script - -We provide a helper script to manage ARM build failure markers: +### Simple File Management +**To skip ARM tests for a module:** ```bash -# Add a failure marker -./scripts/manage-arm-failures.py add fastqc \ - --reason "Package xyz not available for ARM64" \ - --reported-by "your-github-username" \ - --issue-url "https://github.com/nf-core/modules/issues/12345" - -# Check if a module has ARM builds disabled -./scripts/manage-arm-failures.py check fastqc +touch modules/nf-core/problematic-module/.skip-arm +``` -# List all modules with ARM build failures -./scripts/manage-arm-failures.py list +**To link to a wave build failure (optional):** +```bash +echo "https://wave.seqera.io/build/12345" > modules/nf-core/problematic-module/.skip-arm +``` -# Remove a failure marker (re-enable ARM builds) -./scripts/manage-arm-failures.py remove fastqc +**To re-enable ARM tests:** +```bash +rm modules/nf-core/problematic-module/.skip-arm ``` -### Manual Management - -You can also manually create/edit the `.arm-build-failure.yml` files: - -```yaml -# ARM Build Failure Marker -# This file indicates that ARM builds are disabled for this module -# Remove this file to re-enable ARM builds - -reason: "Package XYZ not available for ARM64 architecture" -date: "2024-01-15" -reported_by: "github-username" -issue_url: "https://github.com/nf-core/modules/issues/12345" -error_details: | - The conda package 'some-package' does not have ARM64 builds available. - This causes the wave container build to fail consistently. -notes: | - Alternative solutions investigated: - - Tried using different base image: failed - - Contacted upstream maintainer: waiting for response +**To list modules with ARM tests disabled:** +```bash +find modules/nf-core -name ".skip-arm" ``` ## File Format -The `.arm-build-failure.yml` file supports the following fields: +The `.skip-arm` file can be: +- **Empty**: Just presence of the file disables ARM tests +- **Contain a URL**: Link to wave build failure or GitHub issue +- **Contain notes**: Brief description of the issue -- **`reason`** (required): Brief description of why ARM builds fail -- **`date`** (required): Date when the failure was first reported -- **`reported_by`** (optional): GitHub username of the person reporting the issue -- **`issue_url`** (optional): Link to the related GitHub issue -- **`error_details`** (optional): Detailed error information or logs -- **`notes`** (optional): Additional notes, investigation details, or workarounds attempted +Examples: +```bash +# Empty file +touch modules/nf-core/fastqc/.skip-arm -## Workflow Impact +# With wave build failure link +echo "https://wave.seqera.io/build/12345" > modules/nf-core/fastqc/.skip-arm -### Conda-based Builds (`conda-wave` job) +# With GitHub issue link +echo "https://github.com/nf-core/modules/issues/12345" > modules/nf-core/fastqc/.skip-arm -- Modules with `.arm-build-failure.yml` will skip ARM builds -- Docker and Singularity profiles are both affected -- AMD64 builds continue normally +# With brief note +echo "bioconda package unavailable for ARM64" > modules/nf-core/fastqc/.skip-arm +``` + +## Workflow Impact -### Dockerfile-based Builds (`dockerfile-wave` job) +### Test Filtering -- Modules with `.arm-build-failure.yml` will skip ARM builds -- AMD64 builds continue normally +- Modules with `.skip-arm` will have ARM tests skipped +- AMD64 tests continue normally on all profiles (conda, docker, singularity) +- ARM builds still happen when `environment.yml` files are changed (builds are separate from tests) ### CI/CD Logs -When ARM builds are skipped, you'll see logs like: +When ARM tests are skipped, you'll see logs like: ``` -⚠️ ARM builds disabled for modules/nf-core/fastqc (failure marker found) - Reason: Package XYZ not available for ARM64 architecture - Date: 2024-01-15 - Skipping linux/arm64 build for docker - Skipping linux/arm64 build for singularity +⚠️ Skipping ARM tests for modules/nf-core/fastqc ``` ## Best Practices -### When to Add Failure Markers +### When to Add Skip Markers -- ARM builds consistently fail for legitimate architecture reasons -- The failure is not due to temporary issues (network, quota, etc.) +- ARM tests consistently fail for legitimate architecture reasons +- The failure is due to missing ARM64 packages or architecture-specific issues - The issue has been investigated and documented - An upstream issue has been reported (when applicable) ### Documentation Requirements -When adding a failure marker, please include: -- A clear reason for the failure -- Link to any related GitHub issues -- Details of investigation or workarounds attempted -- Date when the failure was first observed +When adding a skip marker, consider including: +- Link to wave build failure or GitHub issue in the file +- Brief note about the reason (if not obvious from linked issue) ### Regular Review -Periodically review ARM build failures: +Periodically review ARM test skips: - Check if upstream issues have been resolved - Test if packages have become available for ARM64 - Remove markers when issues are fixed -### Re-enabling ARM Builds +### Re-enabling ARM Tests -To re-enable ARM builds for a module: -1. Remove the `.arm-build-failure.yml` file -2. Test the ARM build manually (if possible) -3. Monitor the CI for successful ARM builds +To re-enable ARM tests for a module: +1. Remove the `.skip-arm` file: `rm modules/nf-core/module/.skip-arm` +2. Test the ARM build manually if possible +3. Monitor the CI for successful ARM tests ## Examples ### Example 1: Package Not Available -```yaml -reason: "bioconda package 'tool-xyz' has no ARM64 builds" -date: "2024-01-15" -reported_by: "maintainer-username" -issue_url: "https://github.com/bioconda/bioconda-recipes/issues/12345" -error_details: | - Wave build fails with: - PackagesNotFoundError: The following packages are not available from current channels: - - tool-xyz[version='>=1.0.0',build=*linux-aarch64] -notes: | - Contacted bioconda maintainers about ARM64 support. - Alternative tools investigated but none suitable. +```bash +# Skip ARM tests due to missing bioconda package +echo "https://github.com/bioconda/bioconda-recipes/issues/12345" > modules/nf-core/tool/.skip-arm ``` -### Example 2: Architecture-Specific Compilation Issue - -```yaml -reason: "Compilation fails on ARM64 due to assembly code" -date: "2024-02-01" -reported_by: "developer-username" -issue_url: "https://github.com/upstream/tool/issues/456" -error_details: | - Build fails during compilation with: - Error: unsupported instruction on ARM64 architecture -notes: | - Upstream aware of issue, fix planned for next major release. - No workaround available currently. +### Example 2: Wave Build Failure + +```bash +# Skip ARM tests due to wave container build failure +echo "https://wave.seqera.io/build/abc123def456" > modules/nf-core/tool/.skip-arm ``` -## Troubleshooting +### Example 3: Simple Skip -### Script Not Working +```bash +# Just skip without detailed tracking +touch modules/nf-core/tool/.skip-arm +``` -If the management script doesn't work: -1. Ensure you're in the repository root -2. Check that `uv` is installed -3. Verify the modules directory structure +## Troubleshooting -### Workflow Still Running ARM Builds +### Workflow Still Running ARM Tests -If ARM builds still run despite having a failure marker: -1. Check the file name is exactly `.arm-build-failure.yml` -2. Verify the file is in the correct module directory -3. Ensure the YAML syntax is valid -4. Check the workflow logs for any errors in the filtering step +If ARM tests still run despite having a skip marker: +1. Check the file name is exactly `.skip-arm` +2. Verify the file is in the correct module directory: `modules/nf-core/modulename/.skip-arm` +3. Check the workflow logs for any errors in the filtering step -### Re-enabling Builds +### Re-enabling Tests -To test if ARM builds can be re-enabled: -1. Remove the failure marker file -2. Create a small PR that touches the module's `environment.yml` -3. Monitor the CI workflow for successful ARM builds +To test if ARM tests can be re-enabled: +1. Remove the skip marker file: `rm modules/nf-core/module/.skip-arm` +2. Create a small PR that modifies the module +3. Monitor the CI workflow for successful ARM tests 4. If still failing, re-add the marker with updated information + +### Finding All Skipped Modules + +```bash +# List all modules with ARM tests disabled +find modules/nf-core -name ".skip-arm" -exec dirname {} \; + +# See what's in the skip files +find modules/nf-core -name ".skip-arm" -exec echo "=== {} ===" \; -exec cat {} \; +``` diff --git a/scripts/manage-arm-failures.py b/scripts/manage-arm-failures.py deleted file mode 100644 index 8a3ac5a532ab..000000000000 --- a/scripts/manage-arm-failures.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env uv run -# /// script -# dependencies = [ -# "pyyaml", -# "click", -# ] -# /// - -""" -Helper script to manage ARM build failure markers for nf-core modules. - -This script allows you to: -- Add ARM build failure markers to modules -- Remove ARM build failure markers from modules -- List all modules with ARM build failures -- Check if a specific module has ARM build failures -""" - -import click -import yaml -import os -from pathlib import Path -from datetime import datetime - - -FAILURE_MARKER_FILENAME = ".arm-build-failure.yml" - - -def get_modules_dir(): - """Get the modules directory path.""" - script_dir = Path(__file__).parent - modules_dir = script_dir.parent / "modules" / "nf-core" - if not modules_dir.exists(): - raise click.ClickException(f"Modules directory not found: {modules_dir}") - return modules_dir - - -def get_module_path(module_name): - """Get the path to a specific module.""" - modules_dir = get_modules_dir() - module_path = modules_dir / module_name - if not module_path.exists(): - raise click.ClickException(f"Module not found: {module_name}") - return module_path - - -def get_failure_marker_path(module_name): - """Get the path to the ARM failure marker file for a module.""" - module_path = get_module_path(module_name) - return module_path / FAILURE_MARKER_FILENAME - - -@click.group() -def cli(): - """Manage ARM build failure markers for nf-core modules.""" - pass - - -@cli.command() -@click.argument('module_name') -@click.option('--reason', '-r', required=True, help='Reason for ARM build failure') -@click.option('--reported-by', '-u', help='GitHub username of reporter') -@click.option('--issue-url', '-i', help='URL to related GitHub issue') -@click.option('--error-details', '-e', help='Detailed error information') -@click.option('--notes', '-n', help='Additional notes') -def add(module_name, reason, reported_by, issue_url, error_details, notes): - """Add an ARM build failure marker to a module.""" - failure_marker_path = get_failure_marker_path(module_name) - - if failure_marker_path.exists(): - if not click.confirm(f"ARM failure marker already exists for {module_name}. Overwrite?"): - return - - failure_data = { - 'reason': reason, - 'date': datetime.now().strftime('%Y-%m-%d'), - } - - if reported_by: - failure_data['reported_by'] = reported_by - if issue_url: - failure_data['issue_url'] = issue_url - if error_details: - failure_data['error_details'] = error_details - if notes: - failure_data['notes'] = notes - - with open(failure_marker_path, 'w') as f: - f.write("# ARM Build Failure Marker\n") - f.write("# This file indicates that ARM builds are disabled for this module\n") - f.write("# Remove this file to re-enable ARM builds\n\n") - yaml.dump(failure_data, f, default_flow_style=False, sort_keys=False) - - click.echo(f"✅ Added ARM failure marker for {module_name}") - - -@cli.command() -@click.argument('module_name') -def remove(module_name): - """Remove an ARM build failure marker from a module.""" - failure_marker_path = get_failure_marker_path(module_name) - - if not failure_marker_path.exists(): - click.echo(f"❌ No ARM failure marker found for {module_name}") - return - - if click.confirm(f"Remove ARM failure marker for {module_name}?"): - failure_marker_path.unlink() - click.echo(f"✅ Removed ARM failure marker for {module_name}") - - -@cli.command() -@click.argument('module_name') -def check(module_name): - """Check if a module has an ARM build failure marker.""" - failure_marker_path = get_failure_marker_path(module_name) - - if failure_marker_path.exists(): - click.echo(f"⚠️ {module_name} has ARM builds disabled") - - try: - with open(failure_marker_path, 'r') as f: - failure_data = yaml.safe_load(f) - if failure_data: - click.echo(f" Reason: {failure_data.get('reason', 'Not specified')}") - click.echo(f" Date: {failure_data.get('date', 'Not specified')}") - if 'reported_by' in failure_data: - click.echo(f" Reported by: {failure_data['reported_by']}") - if 'issue_url' in failure_data: - click.echo(f" Issue: {failure_data['issue_url']}") - except Exception as e: - click.echo(f" Error reading failure data: {e}") - else: - click.echo(f"✅ {module_name} has ARM builds enabled") - - -@cli.command() -def list(): - """List all modules with ARM build failure markers.""" - modules_dir = get_modules_dir() - failed_modules = [] - - for module_dir in modules_dir.iterdir(): - if module_dir.is_dir(): - failure_marker = module_dir / FAILURE_MARKER_FILENAME - if failure_marker.exists(): - try: - with open(failure_marker, 'r') as f: - failure_data = yaml.safe_load(f) - failed_modules.append({ - 'name': module_dir.name, - 'reason': failure_data.get('reason', 'Not specified') if failure_data else 'Not specified', - 'date': failure_data.get('date', 'Not specified') if failure_data else 'Not specified' - }) - except Exception as e: - failed_modules.append({ - 'name': module_dir.name, - 'reason': f'Error reading file: {e}', - 'date': 'Unknown' - }) - - if not failed_modules: - click.echo("✅ No modules have ARM build failures") - return - - click.echo(f"⚠️ Found {len(failed_modules)} modules with ARM build failures:\n") - - for module in sorted(failed_modules, key=lambda x: x['name']): - click.echo(f" {module['name']}") - click.echo(f" Reason: {module['reason']}") - click.echo(f" Date: {module['date']}") - click.echo() - - -if __name__ == '__main__': - cli() From 07e379a5d61e3c575d162d150ddb0737838775b1 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 2 Jun 2025 12:48:13 -0500 Subject: [PATCH 04/14] ci: Update ARM test skip marker path in scripts - Changed the path for the `.skip-arm` marker from the module root to the `tests` directory in `filter_arm_tests.py` and GitHub Actions workflows. - Updated documentation in `ARM_BUILD_FAILURES.md` to reflect the new path for skipping ARM tests. - Added a new `.skip-arm` file for the `kraken2` module in the `tests` directory. --- .github/scripts/filter_arm_tests.py | 2 +- .github/workflows/wave.yml | 4 ++-- docs/ARM_BUILD_FAILURES.md | 24 +++++++++---------- modules/nf-core/kraken2/build/tests/.skip-arm | 1 + 4 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 modules/nf-core/kraken2/build/tests/.skip-arm diff --git a/.github/scripts/filter_arm_tests.py b/.github/scripts/filter_arm_tests.py index 09504ed1bfc9..d5e22e4e0794 100755 --- a/.github/scripts/filter_arm_tests.py +++ b/.github/scripts/filter_arm_tests.py @@ -15,7 +15,7 @@ def main(): parts = path.split("/") if len(parts) >= 3 and parts[1] == "nf-core": module_dir = "/".join(parts[:3]) - if os.path.exists(f"{module_dir}/.skip-arm"): + if os.path.exists(f"{module_dir}/tests/.skip-arm"): print(f"⚠️ Skipping ARM tests for {path}", file=sys.stderr) continue filtered.append(path) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index eba3a7f6a9a0..6cff0db68c8c 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -89,7 +89,7 @@ jobs: if: ${{ failure() && matrix.platform == 'linux/arm64' }} run: | MODULE_DIR=$(dirname "${{ matrix.files }}") - SKIP_FILE="$MODULE_DIR/.skip-arm" + SKIP_FILE="$MODULE_DIR/tests/.skip-arm" # Create simple skip marker with workflow run link echo "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" > "$SKIP_FILE" @@ -143,7 +143,7 @@ jobs: if: ${{ failure() && matrix.platform == 'linux/arm64' }} run: | MODULE_DIR=$(dirname "${{ matrix.files }}") - SKIP_FILE="$MODULE_DIR/.skip-arm" + SKIP_FILE="$MODULE_DIR/tests/.skip-arm" # Create simple skip marker with workflow run link echo "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" > "$SKIP_FILE" diff --git a/docs/ARM_BUILD_FAILURES.md b/docs/ARM_BUILD_FAILURES.md index 4b87a2e1ccb0..b251d244239d 100644 --- a/docs/ARM_BUILD_FAILURES.md +++ b/docs/ARM_BUILD_FAILURES.md @@ -35,17 +35,17 @@ The GitHub Actions workflow automatically: **To skip ARM tests for a module:** ```bash -touch modules/nf-core/problematic-module/.skip-arm +touch modules/nf-core/problematic-module/tests/.skip-arm ``` **To link to a wave build failure (optional):** ```bash -echo "https://wave.seqera.io/build/12345" > modules/nf-core/problematic-module/.skip-arm +echo "https://wave.seqera.io/build/12345" > modules/nf-core/problematic-module/tests/.skip-arm ``` **To re-enable ARM tests:** ```bash -rm modules/nf-core/problematic-module/.skip-arm +rm modules/nf-core/problematic-module/tests/.skip-arm ``` **To list modules with ARM tests disabled:** @@ -63,16 +63,16 @@ The `.skip-arm` file can be: Examples: ```bash # Empty file -touch modules/nf-core/fastqc/.skip-arm +touch modules/nf-core/fastqc/tests/.skip-arm # With wave build failure link -echo "https://wave.seqera.io/build/12345" > modules/nf-core/fastqc/.skip-arm +echo "https://wave.seqera.io/build/12345" > modules/nf-core/fastqc/tests/.skip-arm # With GitHub issue link -echo "https://github.com/nf-core/modules/issues/12345" > modules/nf-core/fastqc/.skip-arm +echo "https://github.com/nf-core/modules/issues/12345" > modules/nf-core/fastqc/tests/.skip-arm # With brief note -echo "bioconda package unavailable for ARM64" > modules/nf-core/fastqc/.skip-arm +echo "bioconda package unavailable for ARM64" > modules/nf-core/fastqc/tests/.skip-arm ``` ## Workflow Impact @@ -126,21 +126,21 @@ To re-enable ARM tests for a module: ```bash # Skip ARM tests due to missing bioconda package -echo "https://github.com/bioconda/bioconda-recipes/issues/12345" > modules/nf-core/tool/.skip-arm +echo "https://github.com/bioconda/bioconda-recipes/issues/12345" > modules/nf-core/tool/tests/.skip-arm ``` ### Example 2: Wave Build Failure ```bash # Skip ARM tests due to wave container build failure -echo "https://wave.seqera.io/build/abc123def456" > modules/nf-core/tool/.skip-arm +echo "https://wave.seqera.io/build/abc123def456" > modules/nf-core/tool/tests/.skip-arm ``` ### Example 3: Simple Skip ```bash # Just skip without detailed tracking -touch modules/nf-core/tool/.skip-arm +touch modules/nf-core/tool/tests/.skip-arm ``` ## Troubleshooting @@ -149,13 +149,13 @@ touch modules/nf-core/tool/.skip-arm If ARM tests still run despite having a skip marker: 1. Check the file name is exactly `.skip-arm` -2. Verify the file is in the correct module directory: `modules/nf-core/modulename/.skip-arm` +2. Verify the file is in the correct module tests directory: `modules/nf-core/modulename/tests/.skip-arm` 3. Check the workflow logs for any errors in the filtering step ### Re-enabling Tests To test if ARM tests can be re-enabled: -1. Remove the skip marker file: `rm modules/nf-core/module/.skip-arm` +1. Remove the skip marker file: `rm modules/nf-core/module/tests/.skip-arm` 2. Create a small PR that modifies the module 3. Monitor the CI workflow for successful ARM tests 4. If still failing, re-add the marker with updated information diff --git a/modules/nf-core/kraken2/build/tests/.skip-arm b/modules/nf-core/kraken2/build/tests/.skip-arm new file mode 100644 index 000000000000..f89df22bc02d --- /dev/null +++ b/modules/nf-core/kraken2/build/tests/.skip-arm @@ -0,0 +1 @@ +https://wave.seqera.io/view/builds/bd-c4619b810d01e403_1 \ No newline at end of file From cf2075fc98b4f16db452d666c88cc9fa9587bb37 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 2 Jun 2025 12:49:42 -0500 Subject: [PATCH 05/14] chore: Trigger wave build --- modules/nf-core/kraken2/build/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/kraken2/build/environment.yml b/modules/nf-core/kraken2/build/environment.yml index 19525e8d839f..6ac6ab37b358 100644 --- a/modules/nf-core/kraken2/build/environment.yml +++ b/modules/nf-core/kraken2/build/environment.yml @@ -5,5 +5,5 @@ channels: - bioconda dependencies: - bioconda::kraken2=2.1.5 - - coreutils=9.4 + - conda-forge::coreutils=9.5 - pigz=2.8 From 23a2f2e374a52320ee1f268a3b4812487a700b3e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 2 Jun 2025 14:15:20 -0500 Subject: [PATCH 06/14] chore: Trigger a .arm-skip creation --- modules/nf-core/gtdbtk/classifywf/environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/nf-core/gtdbtk/classifywf/environment.yml b/modules/nf-core/gtdbtk/classifywf/environment.yml index b69848091888..a2ca6c6240db 100644 --- a/modules/nf-core/gtdbtk/classifywf/environment.yml +++ b/modules/nf-core/gtdbtk/classifywf/environment.yml @@ -1,7 +1,6 @@ --- # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json channels: - - conda-forge - bioconda dependencies: - bioconda::gtdbtk=2.4.1 From 8a23c32ff6c99aef1d84ffbe53b45c3b2d1dc36e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 2 Jun 2025 14:22:51 -0500 Subject: [PATCH 07/14] ci: Add ARM skip marker commit step in workflows - Introduced a step to commit and push ARM skip markers when tests fail on the linux/arm64 platform. - Configured Git user details for the bot to handle commits automatically. --- .github/workflows/wave.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 6cff0db68c8c..3c06a06eb95b 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -97,6 +97,15 @@ jobs: echo "Created ARM skip marker at $SKIP_FILE" cat "$SKIP_FILE" + - name: Commit ARM skip marker + if: ${{ failure() && matrix.platform == 'linux/arm64' }} + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git add . + git commit -m "Add ARM test skip marker for $(dirname "${{ matrix.files }}")" || exit 0 + git push + dockerfile-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' @@ -151,6 +160,15 @@ jobs: echo "Created ARM skip marker at $SKIP_FILE" cat "$SKIP_FILE" + - name: Commit ARM skip marker + if: ${{ failure() && matrix.platform == 'linux/arm64' }} + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git add . + git commit -m "Add ARM test skip marker for $(dirname "${{ matrix.files }}")" || exit 0 + git push + # bump-versions: # needs: generate-matrix # name: bump-versions From 5402cf472ce45e70fdbf6b2b59873e55e3663264 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 2 Jun 2025 16:09:36 -0500 Subject: [PATCH 08/14] Revert "chore: Trigger a .arm-skip creation" This reverts commit c4c70ca6b9ba71bb0c4f550b20846eff256b570a. --- modules/nf-core/gtdbtk/classifywf/environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/nf-core/gtdbtk/classifywf/environment.yml b/modules/nf-core/gtdbtk/classifywf/environment.yml index a2ca6c6240db..03c47f521d29 100644 --- a/modules/nf-core/gtdbtk/classifywf/environment.yml +++ b/modules/nf-core/gtdbtk/classifywf/environment.yml @@ -1,6 +1,8 @@ --- # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json channels: + - conda-forge - bioconda dependencies: - bioconda::gtdbtk=2.4.1 + - cowpy From 334964e71c9b0d8631e5fb175069c060c0099bc9 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 2 Jun 2025 16:14:14 -0500 Subject: [PATCH 09/14] ci: Remove more filtering from wave --- .github/workflows/wave.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 3c06a06eb95b..f379624857be 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -26,8 +26,6 @@ jobs: outputs: conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }} dockerfile-matrix: ${{ steps.docker-diff.outputs.all_changed_files }} - filtered-conda-matrix: ${{ steps.filter-conda.outputs.result }} - filtered-dockerfile-matrix: ${{ steps.filter-dockerfile.outputs.result }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -52,7 +50,7 @@ jobs: conda-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: ${{ needs.generate-matrix.outputs.filtered-conda-matrix != '' && needs.generate-matrix.outputs.filtered-conda-matrix != '{"include":[]}' }} + if: ${{ needs.generate-matrix.outputs.conda-matrix != '' && needs.generate-matrix.outputs.conda-matrix != '{"include":[]}' }} needs: generate-matrix name: Build ${{ matrix.profile }} ${{ matrix.platform }} Container runs-on: ubuntu-latest @@ -60,7 +58,7 @@ jobs: strategy: fail-fast: false max-parallel: 4 - matrix: ${{ fromJson(needs.generate-matrix.outputs.filtered-conda-matrix) }} + matrix: ${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -109,7 +107,7 @@ jobs: dockerfile-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: ${{ needs.generate-matrix.outputs.filtered-dockerfile-matrix != '' && needs.generate-matrix.outputs.filtered-dockerfile-matrix != '{"include":[]}' }} + if: ${{ needs.generate-matrix.outputs.dockerfile-matrix != '' && needs.generate-matrix.outputs.dockerfile-matrix != '{"include":[]}' }} needs: generate-matrix name: Build Dockerfile-based ${{ matrix.platform }} Container runs-on: ubuntu-latest @@ -117,7 +115,7 @@ jobs: strategy: fail-fast: false max-parallel: 4 - matrix: ${{ fromJson(needs.generate-matrix.outputs.filtered-dockerfile-matrix) }} + matrix: ${{ fromJson(needs.generate-matrix.outputs.dockerfile-matrix) }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 From 37e652c9077e91a8c70656f625df15f4944e6f6f Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 3 Jun 2025 09:50:57 -0500 Subject: [PATCH 10/14] refactor: Use arm_failure tags in nf-test files - Introduced `add_arm_failure_tag.py` script to append the `arm_failure` tag to nf-test files for modules that fail on ARM architecture. - Updated GitHub workflows to utilize the new script for tagging during ARM test failures. - Removed obsolete `.skip-arm` files in favor of the new tagging system. --- .github/scripts/add_arm_failure_tag.py | 67 +++++++++++++++++++ .github/scripts/filter_arm_tests.py | 25 ------- .github/workflows/nf-test.yml | 18 ++--- .github/workflows/wave.yml | 24 +++---- modules/nf-core/kraken2/build/tests/.skip-arm | 1 - .../nf-core/kraken2/build/tests/main.nf.test | 3 +- 6 files changed, 81 insertions(+), 57 deletions(-) create mode 100755 .github/scripts/add_arm_failure_tag.py delete mode 100755 .github/scripts/filter_arm_tests.py delete mode 100644 modules/nf-core/kraken2/build/tests/.skip-arm diff --git a/.github/scripts/add_arm_failure_tag.py b/.github/scripts/add_arm_failure_tag.py new file mode 100755 index 000000000000..7e13fe57a005 --- /dev/null +++ b/.github/scripts/add_arm_failure_tag.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Add arm_failure tag to nf-test files for modules that fail on ARM""" + +import sys +import re +from pathlib import Path + +def add_arm_failure_tag(test_file_path): + """Add arm_failure tag to the test file if it doesn't already exist""" + + if not test_file_path.exists(): + print(f"Test file not found: {test_file_path}") + return False + + content = test_file_path.read_text() + + # Check if arm_failure tag already exists + if 'tag "arm_failure"' in content: + print(f"ARM failure tag already exists in {test_file_path}") + return False + + # Find the last tag line in the file - looking for pattern: tag "something" + tag_pattern = r'(\s*tag\s+"[^"]+"\s*\n)' + tag_matches = list(re.finditer(tag_pattern, content)) + + if tag_matches: + # Get the last tag match + last_tag = tag_matches[-1] + # Insert the new tag after the last existing tag with proper formatting + before = content[:last_tag.end()] + after = content[last_tag.end():] + new_content = before + ' tag "arm_failure"\n\n' + after + else: + # No existing tags, add after the process line + process_pattern = r'(process\s+"[^"]+"\s*\n)' + process_match = re.search(process_pattern, content) + if process_match: + before = content[:process_match.end()] + after = content[process_match.end():] + new_content = before + '\n tag "arm_failure"\n' + after + else: + print(f"Could not find process line in {test_file_path}") + return False + + # Write the updated content + test_file_path.write_text(new_content) + print(f"Added ARM failure tag to {test_file_path}") + return True + +def main(): + if len(sys.argv) < 2: + print("Usage: add_arm_failure_tag.py [module_path2] ...") + print("Example: add_arm_failure_tag.py modules/nf-core/fastqc") + sys.exit(1) + + success_count = 0 + for module_path in sys.argv[1:]: + module_dir = Path(module_path) + test_file = module_dir / "tests" / "main.nf.test" + + if add_arm_failure_tag(test_file): + success_count += 1 + + print(f"Successfully added ARM failure tags to {success_count} module(s)") + +if __name__ == "__main__": + main() diff --git a/.github/scripts/filter_arm_tests.py b/.github/scripts/filter_arm_tests.py deleted file mode 100755 index d5e22e4e0794..000000000000 --- a/.github/scripts/filter_arm_tests.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -"""Skip ARM tests for modules with .skip-arm marker files""" -import json, sys, os - -def main(): - if len(sys.argv) != 2: - print(json.dumps([])) - return - - paths = json.loads(sys.argv[1]) - filtered = [] - - for path in paths: - # Extract module dir (modules/nf-core/xxx or subworkflows/nf-core/xxx) - parts = path.split("/") - if len(parts) >= 3 and parts[1] == "nf-core": - module_dir = "/".join(parts[:3]) - if os.path.exists(f"{module_dir}/tests/.skip-arm"): - print(f"⚠️ Skipping ARM tests for {path}", file=sys.stderr) - continue - filtered.append(path) - - print(json.dumps(filtered)) - -if __name__ == "__main__": main() diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 27708101a5f4..9be22060b1c2 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -62,9 +62,9 @@ jobs: head: ${{ github.sha }} base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} n_parents: 0 - exclude_tags: "gpu" + exclude_tags: "gpu,arm_failure" - # FIXME Can't we just do this with nf-test --filter? + # FIXME Can't we just do this with nf-test --filter? - name: Separate modules and subworkflows id: components run: | @@ -110,7 +110,7 @@ jobs: NXF_ANSI_LOG: false TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} outputs: - arm_filtered_paths: ${{ steps.filter-arm.outputs.arm_filtered_paths }} + filtered_paths: ${{ steps.filter.outputs.filtered_paths }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -153,20 +153,12 @@ jobs: echo "filtered_paths=${FILTERED}" >> $GITHUB_OUTPUT - - name: Filter ARM tests - id: filter-arm - run: | - PATHS='${{ steps.filter.outputs.filtered_paths }}' - ARM_FILTERED=$(.github/scripts/filter_arm_tests.py "$PATHS") - echo "arm_filtered_paths=${ARM_FILTERED}" >> $GITHUB_OUTPUT - - name: debug run: | echo filtered_paths = ${{ steps.filter.outputs.filtered_paths }} - echo arm_filtered_paths = ${{ steps.filter-arm.outputs.arm_filtered_paths }} - name: Run nf-test Action - if: ${{steps.filter-arm.outputs.arm_filtered_paths != '[]'}} + if: ${{steps.filter.outputs.filtered_paths != '[]'}} uses: ./.github/actions/nf-test-action env: SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} @@ -177,7 +169,7 @@ jobs: profile: ${{ matrix.profile }}${{ runner.arch == 'ARM64' && ',arm_CI,wave' || '' }} shard: ${{ matrix.shard }} total_shards: ${{ env.TOTAL_SHARDS }} - paths: "${{ join(fromJson(steps.filter-arm.outputs.arm_filtered_paths), ' ') }}" + paths: "${{ join(fromJson(steps.filter.outputs.filtered_paths), ' ') }}" confirm-pass-nf-test: runs-on: diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index f379624857be..bce9bc0bccdc 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -87,21 +87,17 @@ jobs: if: ${{ failure() && matrix.platform == 'linux/arm64' }} run: | MODULE_DIR=$(dirname "${{ matrix.files }}") - SKIP_FILE="$MODULE_DIR/tests/.skip-arm" - # Create simple skip marker with workflow run link - echo "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" > "$SKIP_FILE" + # Add arm_failure tag to nf-test file + python .github/scripts/add_arm_failure_tag.py "$MODULE_DIR" - echo "Created ARM skip marker at $SKIP_FILE" - cat "$SKIP_FILE" - - - name: Commit ARM skip marker + - name: Commit ARM failure tag if: ${{ failure() && matrix.platform == 'linux/arm64' }} run: | git config user.email "core@nf-co.re" git config user.name "nf-core-bot" git add . - git commit -m "Add ARM test skip marker for $(dirname "${{ matrix.files }}")" || exit 0 + git commit -m "Add ARM failure tag for $(dirname "${{ matrix.files }}")" || exit 0 git push dockerfile-wave: @@ -150,21 +146,17 @@ jobs: if: ${{ failure() && matrix.platform == 'linux/arm64' }} run: | MODULE_DIR=$(dirname "${{ matrix.files }}") - SKIP_FILE="$MODULE_DIR/tests/.skip-arm" - - # Create simple skip marker with workflow run link - echo "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" > "$SKIP_FILE" - echo "Created ARM skip marker at $SKIP_FILE" - cat "$SKIP_FILE" + # Add arm_failure tag to nf-test file + python .github/scripts/add_arm_failure_tag.py "$MODULE_DIR" - - name: Commit ARM skip marker + - name: Commit ARM failure tag if: ${{ failure() && matrix.platform == 'linux/arm64' }} run: | git config user.email "core@nf-co.re" git config user.name "nf-core-bot" git add . - git commit -m "Add ARM test skip marker for $(dirname "${{ matrix.files }}")" || exit 0 + git commit -m "Add ARM failure tag for $(dirname "${{ matrix.files }}")" || exit 0 git push # bump-versions: diff --git a/modules/nf-core/kraken2/build/tests/.skip-arm b/modules/nf-core/kraken2/build/tests/.skip-arm deleted file mode 100644 index f89df22bc02d..000000000000 --- a/modules/nf-core/kraken2/build/tests/.skip-arm +++ /dev/null @@ -1 +0,0 @@ -https://wave.seqera.io/view/builds/bd-c4619b810d01e403_1 \ No newline at end of file diff --git a/modules/nf-core/kraken2/build/tests/main.nf.test b/modules/nf-core/kraken2/build/tests/main.nf.test index fbeb91a827e3..55f5d0cb3193 100644 --- a/modules/nf-core/kraken2/build/tests/main.nf.test +++ b/modules/nf-core/kraken2/build/tests/main.nf.test @@ -10,6 +10,7 @@ nextflow_process { tag "gunzip" tag "modules" tag "modules_nfcore" + tag "arm_failure" setup { @@ -128,8 +129,6 @@ nextflow_process { test("sarscov2 protein_db stub") { - tag "arm_failure" - options "-stub" setup { From 963304ff9f93d856ba7baa1d2b208f424dad10bb Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 3 Jun 2025 10:08:31 -0500 Subject: [PATCH 11/14] style: Touch a module that works with arm --- .github/workflows/nf-test.yml | 3 +- modules/nf-core/bowtie2/align/main.nf | 64 +++++++++++++++------------ 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 9be22060b1c2..f2fede36b449 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -104,8 +104,7 @@ jobs: profile: [conda, docker, singularity] arch: - x64 - # FIXME - # - arm64 + - arm64 env: NXF_ANSI_LOG: false TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} diff --git a/modules/nf-core/bowtie2/align/main.nf b/modules/nf-core/bowtie2/align/main.nf index 631d0bf79c2e..762a05838349 100644 --- a/modules/nf-core/bowtie2/align/main.nf +++ b/modules/nf-core/bowtie2/align/main.nf @@ -1,28 +1,28 @@ process BOWTIE2_ALIGN { - tag "$meta.id" + tag "${meta.id}" label 'process_high' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/b4/b41b403e81883126c3227fc45840015538e8e2212f13abc9ae84e4b98891d51c/data' : - 'community.wave.seqera.io/library/bowtie2_htslib_samtools_pigz:edeb13799090a2a6' }" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/b4/b41b403e81883126c3227fc45840015538e8e2212f13abc9ae84e4b98891d51c/data' + : 'community.wave.seqera.io/library/bowtie2_htslib_samtools_pigz:edeb13799090a2a6'}" input: - tuple val(meta) , path(reads) + tuple val(meta), path(reads) tuple val(meta2), path(index) tuple val(meta3), path(fasta) - val save_unaligned - val sort_bam + val save_unaligned + val sort_bam output: - tuple val(meta), path("*.sam") , emit: sam , optional:true - tuple val(meta), path("*.bam") , emit: bam , optional:true - tuple val(meta), path("*.cram") , emit: cram , optional:true - tuple val(meta), path("*.csi") , emit: csi , optional:true - tuple val(meta), path("*.crai") , emit: crai , optional:true - tuple val(meta), path("*.log") , emit: log - tuple val(meta), path("*fastq.gz") , emit: fastq , optional:true - path "versions.yml" , emit: versions + tuple val(meta), path("*.sam"), emit: sam, optional: true + tuple val(meta), path("*.bam"), emit: bam, optional: true + tuple val(meta), path("*.cram"), emit: cram, optional: true + tuple val(meta), path("*.csi"), emit: csi, optional: true + tuple val(meta), path("*.crai"), emit: crai, optional: true + tuple val(meta), path("*.log"), emit: log + tuple val(meta), path("*fastq.gz"), emit: fastq, optional: true + path "versions.yml", emit: versions when: task.ext.when == null || task.ext.when @@ -37,17 +37,20 @@ process BOWTIE2_ALIGN { if (meta.single_end) { unaligned = save_unaligned ? "--un-gz ${prefix}.unmapped.fastq.gz" : "" reads_args = "-U ${reads}" - } else { + } + else { unaligned = save_unaligned ? "--un-conc-gz ${prefix}.unmapped.fastq.gz" : "" reads_args = "-1 ${reads[0]} -2 ${reads[1]}" } def samtools_command = sort_bam ? 'sort' : 'view' def extension_pattern = /(--output-fmt|-O)+\s+(\S+)/ - def extension_matcher = (args2 =~ extension_pattern) + def extension_matcher = (args2 =~ extension_pattern) def extension = extension_matcher.getCount() > 0 ? extension_matcher[0][2].toLowerCase() : "bam" - def reference = fasta && extension=="cram" ? "--reference ${fasta}" : "" - if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" + def reference = fasta && extension == "cram" ? "--reference ${fasta}" : "" + if (!fasta && extension == "cram") { + error("Fasta reference is required for CRAM output") + } """ INDEX=`find -L ./ -name "*.rev.1.bt2" | sed "s/\\.rev.1.bt2\$//"` @@ -56,12 +59,12 @@ process BOWTIE2_ALIGN { bowtie2 \\ -x \$INDEX \\ - $reads_args \\ - --threads $task.cpus \\ - $unaligned \\ - $args \\ + ${reads_args} \\ + --threads ${task.cpus} \\ + ${unaligned} \\ + ${args} \\ 2>| >(tee ${prefix}.bowtie2.log >&2) \\ - | samtools $samtools_command $args2 --threads $task.cpus ${reference} -o ${prefix}.${extension} - + | samtools ${samtools_command} ${args2} --threads ${task.cpus} ${reference} -o ${prefix}.${extension} - if [ -f ${prefix}.unmapped.fastq.1.gz ]; then mv ${prefix}.unmapped.fastq.1.gz ${prefix}.unmapped_1.fastq.gz @@ -83,19 +86,23 @@ process BOWTIE2_ALIGN { def args2 = task.ext.args2 ?: "" def prefix = task.ext.prefix ?: "${meta.id}" def extension_pattern = /(--output-fmt|-O)+\s+(\S+)/ - def extension = (args2 ==~ extension_pattern) ? (args2 =~ extension_pattern)[0][2].toLowerCase() : "bam" + def extension = args2 ==~ extension_pattern ? (args2 =~ extension_pattern)[0][2].toLowerCase() : "bam" def create_unmapped = "" if (meta.single_end) { create_unmapped = save_unaligned ? "touch ${prefix}.unmapped.fastq.gz" : "" - } else { + } + else { create_unmapped = save_unaligned ? "touch ${prefix}.unmapped_1.fastq.gz && touch ${prefix}.unmapped_2.fastq.gz" : "" } - if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" + if (!fasta && extension == "cram") { + error("Fasta reference is required for CRAM output") + } def create_index = "" if (extension == "cram") { create_index = "touch ${prefix}.crai" - } else if (extension == "bam") { + } + else if (extension == "bam") { create_index = "touch ${prefix}.csi" } @@ -112,5 +119,4 @@ process BOWTIE2_ALIGN { pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) END_VERSIONS """ - } From bdee3317526ee231d6f3a208c8f7b92e376e9115 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 3 Jun 2025 10:56:25 -0500 Subject: [PATCH 12/14] chore: Manually add arm failures for subworkflows --- .../nf-core/fasta_build_add_kraken2/tests/main.nf.test | 3 ++- .../nf-core/fasta_build_add_kraken2_bracken/tests/main.nf.test | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/fasta_build_add_kraken2/tests/main.nf.test b/subworkflows/nf-core/fasta_build_add_kraken2/tests/main.nf.test index cfcca39878d0..83783c70f9f2 100644 --- a/subworkflows/nf-core/fasta_build_add_kraken2/tests/main.nf.test +++ b/subworkflows/nf-core/fasta_build_add_kraken2/tests/main.nf.test @@ -11,6 +11,7 @@ nextflow_workflow { tag "kraken2" tag "kraken2/add" tag "kraken2/build" + tag "arm_failure" test("metagenome - fasta - nocleanup") { @@ -154,4 +155,4 @@ nextflow_workflow { } } -} +} \ No newline at end of file diff --git a/subworkflows/nf-core/fasta_build_add_kraken2_bracken/tests/main.nf.test b/subworkflows/nf-core/fasta_build_add_kraken2_bracken/tests/main.nf.test index e036a04e5043..e250b3e5f026 100644 --- a/subworkflows/nf-core/fasta_build_add_kraken2_bracken/tests/main.nf.test +++ b/subworkflows/nf-core/fasta_build_add_kraken2_bracken/tests/main.nf.test @@ -12,6 +12,7 @@ nextflow_workflow { tag "kraken2/add" tag "kraken2/build" tag "bracken/build" + tag "arm_failure" test("metagenome - nocleanup - nobracken - fasta") { @@ -212,4 +213,4 @@ nextflow_workflow { ) } } -} +} \ No newline at end of file From 3d8a24e53db09c18e08520f6e6d1f180cb2e3af9 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 3 Jun 2025 11:21:29 -0500 Subject: [PATCH 13/14] chore: Remove experiments --- .github/workflows/wave.yml | 2 +- modules/nf-core/fastqc/.arm-build-failure.yml | 7 ------- modules/nf-core/gtdbtk/classifywf/environment.yml | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 modules/nf-core/fastqc/.arm-build-failure.yml diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index bce9bc0bccdc..9894d350e6b6 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -71,7 +71,7 @@ jobs: - name: Build ${{ matrix.profile }} container # FIXME Hack while iron out the CI - continue-on-error: false + continue-on-error: true env: PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }} run: | diff --git a/modules/nf-core/fastqc/.arm-build-failure.yml b/modules/nf-core/fastqc/.arm-build-failure.yml deleted file mode 100644 index d2dfab265120..000000000000 --- a/modules/nf-core/fastqc/.arm-build-failure.yml +++ /dev/null @@ -1,7 +0,0 @@ -# ARM Build Failure Marker -# This file indicates that ARM builds are disabled for this module -# Remove this file to re-enable ARM builds - -reason: Testing ARM failure filtering -date: '2025-06-02' -reported_by: test-user diff --git a/modules/nf-core/gtdbtk/classifywf/environment.yml b/modules/nf-core/gtdbtk/classifywf/environment.yml index 03c47f521d29..b69848091888 100644 --- a/modules/nf-core/gtdbtk/classifywf/environment.yml +++ b/modules/nf-core/gtdbtk/classifywf/environment.yml @@ -5,4 +5,3 @@ channels: - bioconda dependencies: - bioconda::gtdbtk=2.4.1 - - cowpy From 7344705fe32af1716d5fe085ed797627af8ccce0 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 3 Jun 2025 11:53:21 -0500 Subject: [PATCH 14/14] test: Mark arm_failures --- subworkflows/nf-core/fastq_align_dna/tests/main.nf.test | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test index 48988102919a..1a5ac2ed1bfb 100644 --- a/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test @@ -235,6 +235,9 @@ nextflow_workflow { } } test ("test_fastq_align_dragmap_SE"){ + + tag "arm_failure" + setup { run("DRAGMAP_HASHTABLE") { script "../../../../modules/nf-core/dragmap/hashtable/main.nf" @@ -269,6 +272,9 @@ nextflow_workflow { } } test ("test_fastq_align_dragmap_PE"){ + + tag "arm_failure" + setup { run("DRAGMAP_HASHTABLE") { script "../../../../modules/nf-core/dragmap/hashtable/main.nf" @@ -333,6 +339,7 @@ nextflow_workflow { } } test ("test_fastq_align_snapaligner_PE"){ + tag "arm_failure" setup { run("SNAPALIGNER_INDEX") { script "../../../../modules/nf-core/snapaligner/index/main.nf" @@ -362,4 +369,4 @@ nextflow_workflow { ) } } -} +} \ No newline at end of file