diff --git a/.github/scripts/add_arm_failure_tag.py b/.github/scripts/add_arm_failure_tag.py new file mode 100755 index 000000000000..7e13fe57a005 --- /dev/null +++ b/.github/scripts/add_arm_failure_tag.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Add arm_failure tag to nf-test files for modules that fail on ARM""" + +import sys +import re +from pathlib import Path + +def add_arm_failure_tag(test_file_path): + """Add arm_failure tag to the test file if it doesn't already exist""" + + if not test_file_path.exists(): + print(f"Test file not found: {test_file_path}") + return False + + content = test_file_path.read_text() + + # Check if arm_failure tag already exists + if 'tag "arm_failure"' in content: + print(f"ARM failure tag already exists in {test_file_path}") + return False + + # Find the last tag line in the file - looking for pattern: tag "something" + tag_pattern = r'(\s*tag\s+"[^"]+"\s*\n)' + tag_matches = list(re.finditer(tag_pattern, content)) + + if tag_matches: + # Get the last tag match + last_tag = tag_matches[-1] + # Insert the new tag after the last existing tag with proper formatting + before = content[:last_tag.end()] + after = content[last_tag.end():] + new_content = before + ' tag "arm_failure"\n\n' + after + else: + # No existing tags, add after the process line + process_pattern = r'(process\s+"[^"]+"\s*\n)' + process_match = re.search(process_pattern, content) + if process_match: + before = content[:process_match.end()] + after = content[process_match.end():] + new_content = before + '\n tag "arm_failure"\n' + after + else: + print(f"Could not find process line in {test_file_path}") + return False + + # Write the updated content + test_file_path.write_text(new_content) + print(f"Added ARM failure tag to {test_file_path}") + return True + +def main(): + if len(sys.argv) < 2: + print("Usage: add_arm_failure_tag.py [module_path2] ...") + print("Example: add_arm_failure_tag.py modules/nf-core/fastqc") + sys.exit(1) + + success_count = 0 + for module_path in sys.argv[1:]: + module_dir = Path(module_path) + test_file = module_dir / "tests" / "main.nf.test" + + if add_arm_failure_tag(test_file): + success_count += 1 + + print(f"Successfully added ARM failure tags to {success_count} module(s)") + +if __name__ == "__main__": + main() diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 51e27840dbff..f2fede36b449 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -62,8 +62,9 @@ jobs: head: ${{ github.sha }} base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} n_parents: 0 - exclude_tags: "gpu" + exclude_tags: "gpu,arm_failure" + # FIXME Can't we just do this with nf-test --filter? - name: Separate modules and subworkflows id: components run: | @@ -103,8 +104,7 @@ jobs: profile: [conda, docker, singularity] arch: - x64 - # FIXME - # - arm64 + - arm64 env: NXF_ANSI_LOG: false TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} @@ -158,7 +158,6 @@ jobs: - name: Run nf-test Action if: ${{steps.filter.outputs.filtered_paths != '[]'}} - continue-on-error: ${{ runner.arch == 'ARM64' }} uses: ./.github/actions/nf-test-action env: SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} @@ -188,7 +187,7 @@ jobs: run: exit 1 - name: If no tests run, but filtered_paths was not empty and no tests succeeded, fail - if: ${{ needs.nf-test.outputs.filtered_paths != '' && !contains(needs.*.result, 'success') }} + if: ${{ needs.nf-test.outputs.arm_filtered_paths != '' && !contains(needs.*.result, 'success') }} run: exit 1 - name: All tests ok @@ -198,10 +197,10 @@ jobs: - name: debug-print if: always() run: | - echo "::group::DEBUG: `needs` Contents" + echo "::group::DEBUG: \`needs\` Contents" echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}" echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" - echo "DEBUG: needs.nf-test.outputs.filtered_paths = ${{ needs.nf-test.outputs.filtered_paths }}" + echo "DEBUG: needs.nf-test.outputs.arm_filtered_paths = ${{ needs.nf-test.outputs.arm_filtered_paths }}" echo "::endgroup::" - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 7297b30b1e92..9894d350e6b6 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -50,7 +50,7 @@ jobs: conda-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }} + if: ${{ needs.generate-matrix.outputs.conda-matrix != '' && needs.generate-matrix.outputs.conda-matrix != '{"include":[]}' }} needs: generate-matrix name: Build ${{ matrix.profile }} ${{ matrix.platform }} Container runs-on: ubuntu-latest @@ -58,10 +58,8 @@ jobs: strategy: fail-fast: false max-parallel: 4 - matrix: - files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}" - profile: [docker, singularity] - platform: ["linux/amd64", "linux/arm64"] + matrix: ${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }} + steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -85,10 +83,27 @@ jobs: --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} + - name: Mark ARM build failure + if: ${{ failure() && matrix.platform == 'linux/arm64' }} + run: | + MODULE_DIR=$(dirname "${{ matrix.files }}") + + # Add arm_failure tag to nf-test file + python .github/scripts/add_arm_failure_tag.py "$MODULE_DIR" + + - name: Commit ARM failure tag + if: ${{ failure() && matrix.platform == 'linux/arm64' }} + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git add . + git commit -m "Add ARM failure tag for $(dirname "${{ matrix.files }}")" || exit 0 + git push + dockerfile-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: ${{ needs.generate-matrix.outputs.dockerfile-matrix != '[]' }} + if: ${{ needs.generate-matrix.outputs.dockerfile-matrix != '' && needs.generate-matrix.outputs.dockerfile-matrix != '{"include":[]}' }} needs: generate-matrix name: Build Dockerfile-based ${{ matrix.platform }} Container runs-on: ubuntu-latest @@ -96,12 +111,7 @@ jobs: strategy: fail-fast: false max-parallel: 4 - matrix: - files: "${{ fromJson(needs.generate-matrix.outputs.dockerfile-matrix) }}" - # NOTE singularity build requires a Singularity definition file in place of Dockerfile - # https://nextflow.slack.com/archives/C0477AS31T5/p1731755350230149?thread_ts=1731697852.661849&cid=C0477AS31T5 - # profile: [docker] - platform: ["linux/amd64", "linux/arm64"] + matrix: ${{ fromJson(needs.generate-matrix.outputs.dockerfile-matrix) }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -132,6 +142,23 @@ jobs: --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} + - name: Mark ARM build failure + if: ${{ failure() && matrix.platform == 'linux/arm64' }} + run: | + MODULE_DIR=$(dirname "${{ matrix.files }}") + + # Add arm_failure tag to nf-test file + python .github/scripts/add_arm_failure_tag.py "$MODULE_DIR" + + - name: Commit ARM failure tag + if: ${{ failure() && matrix.platform == 'linux/arm64' }} + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git add . + git commit -m "Add ARM failure tag for $(dirname "${{ matrix.files }}")" || exit 0 + git push + # bump-versions: # needs: generate-matrix # name: bump-versions diff --git a/docs/ARM_BUILD_FAILURES.md b/docs/ARM_BUILD_FAILURES.md new file mode 100644 index 000000000000..b251d244239d --- /dev/null +++ b/docs/ARM_BUILD_FAILURES.md @@ -0,0 +1,171 @@ +# ARM Test Skipping + +This document describes the simple system for skipping ARM tests in nf-core modules. + +## Overview + +Some modules may fail to run tests on ARM64 architecture due to various reasons such as: +- Dependencies not available for ARM64 +- Architecture-specific compilation issues +- Upstream package limitations + +To handle these cases gracefully, we have implemented a simple system that allows temporarily disabling ARM tests for specific modules until the issues are resolved. + +## How It Works + +### Skip Marker Files + +When a module consistently fails ARM tests, you can create a `.skip-arm` file in the module directory. This file: + +1. **Disables ARM tests** for that module in CI/CD pipelines +2. **Can contain a link** to the wave build failure (optional) +3. **Can be easily removed** when the issue is resolved + +### Workflow Integration + +The GitHub Actions workflow automatically: +- Checks for `.skip-arm` files before running tests on ARM runners +- Skips ARM tests for modules with skip markers +- Continues with AMD64 tests as normal +- Logs which tests are being skipped + +## Managing ARM Test Skipping + +### Simple File Management + +**To skip ARM tests for a module:** +```bash +touch modules/nf-core/problematic-module/tests/.skip-arm +``` + +**To link to a wave build failure (optional):** +```bash +echo "https://wave.seqera.io/build/12345" > modules/nf-core/problematic-module/tests/.skip-arm +``` + +**To re-enable ARM tests:** +```bash +rm modules/nf-core/problematic-module/tests/.skip-arm +``` + +**To list modules with ARM tests disabled:** +```bash +find modules/nf-core -name ".skip-arm" +``` + +## File Format + +The `.skip-arm` file can be: +- **Empty**: Just presence of the file disables ARM tests +- **Contain a URL**: Link to wave build failure or GitHub issue +- **Contain notes**: Brief description of the issue + +Examples: +```bash +# Empty file +touch modules/nf-core/fastqc/tests/.skip-arm + +# With wave build failure link +echo "https://wave.seqera.io/build/12345" > modules/nf-core/fastqc/tests/.skip-arm + +# With GitHub issue link +echo "https://github.com/nf-core/modules/issues/12345" > modules/nf-core/fastqc/tests/.skip-arm + +# With brief note +echo "bioconda package unavailable for ARM64" > modules/nf-core/fastqc/tests/.skip-arm +``` + +## Workflow Impact + +### Test Filtering + +- Modules with `.skip-arm` will have ARM tests skipped +- AMD64 tests continue normally on all profiles (conda, docker, singularity) +- ARM builds still happen when `environment.yml` files are changed (builds are separate from tests) + +### CI/CD Logs + +When ARM tests are skipped, you'll see logs like: + +``` +⚠️ Skipping ARM tests for modules/nf-core/fastqc +``` + +## Best Practices + +### When to Add Skip Markers + +- ARM tests consistently fail for legitimate architecture reasons +- The failure is due to missing ARM64 packages or architecture-specific issues +- The issue has been investigated and documented +- An upstream issue has been reported (when applicable) + +### Documentation Requirements + +When adding a skip marker, consider including: +- Link to wave build failure or GitHub issue in the file +- Brief note about the reason (if not obvious from linked issue) + +### Regular Review + +Periodically review ARM test skips: +- Check if upstream issues have been resolved +- Test if packages have become available for ARM64 +- Remove markers when issues are fixed + +### Re-enabling ARM Tests + +To re-enable ARM tests for a module: +1. Remove the `.skip-arm` file: `rm modules/nf-core/module/.skip-arm` +2. Test the ARM build manually if possible +3. Monitor the CI for successful ARM tests + +## Examples + +### Example 1: Package Not Available + +```bash +# Skip ARM tests due to missing bioconda package +echo "https://github.com/bioconda/bioconda-recipes/issues/12345" > modules/nf-core/tool/tests/.skip-arm +``` + +### Example 2: Wave Build Failure + +```bash +# Skip ARM tests due to wave container build failure +echo "https://wave.seqera.io/build/abc123def456" > modules/nf-core/tool/tests/.skip-arm +``` + +### Example 3: Simple Skip + +```bash +# Just skip without detailed tracking +touch modules/nf-core/tool/tests/.skip-arm +``` + +## Troubleshooting + +### Workflow Still Running ARM Tests + +If ARM tests still run despite having a skip marker: +1. Check the file name is exactly `.skip-arm` +2. Verify the file is in the correct module tests directory: `modules/nf-core/modulename/tests/.skip-arm` +3. Check the workflow logs for any errors in the filtering step + +### Re-enabling Tests + +To test if ARM tests can be re-enabled: +1. Remove the skip marker file: `rm modules/nf-core/module/tests/.skip-arm` +2. Create a small PR that modifies the module +3. Monitor the CI workflow for successful ARM tests +4. If still failing, re-add the marker with updated information + +### Finding All Skipped Modules + +```bash +# List all modules with ARM tests disabled +find modules/nf-core -name ".skip-arm" -exec dirname {} \; + +# See what's in the skip files +find modules/nf-core -name ".skip-arm" -exec echo "=== {} ===" \; -exec cat {} \; +``` diff --git a/modules/nf-core/bowtie2/align/main.nf b/modules/nf-core/bowtie2/align/main.nf index 631d0bf79c2e..762a05838349 100644 --- a/modules/nf-core/bowtie2/align/main.nf +++ b/modules/nf-core/bowtie2/align/main.nf @@ -1,28 +1,28 @@ process BOWTIE2_ALIGN { - tag "$meta.id" + tag "${meta.id}" label 'process_high' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/b4/b41b403e81883126c3227fc45840015538e8e2212f13abc9ae84e4b98891d51c/data' : - 'community.wave.seqera.io/library/bowtie2_htslib_samtools_pigz:edeb13799090a2a6' }" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/b4/b41b403e81883126c3227fc45840015538e8e2212f13abc9ae84e4b98891d51c/data' + : 'community.wave.seqera.io/library/bowtie2_htslib_samtools_pigz:edeb13799090a2a6'}" input: - tuple val(meta) , path(reads) + tuple val(meta), path(reads) tuple val(meta2), path(index) tuple val(meta3), path(fasta) - val save_unaligned - val sort_bam + val save_unaligned + val sort_bam output: - tuple val(meta), path("*.sam") , emit: sam , optional:true - tuple val(meta), path("*.bam") , emit: bam , optional:true - tuple val(meta), path("*.cram") , emit: cram , optional:true - tuple val(meta), path("*.csi") , emit: csi , optional:true - tuple val(meta), path("*.crai") , emit: crai , optional:true - tuple val(meta), path("*.log") , emit: log - tuple val(meta), path("*fastq.gz") , emit: fastq , optional:true - path "versions.yml" , emit: versions + tuple val(meta), path("*.sam"), emit: sam, optional: true + tuple val(meta), path("*.bam"), emit: bam, optional: true + tuple val(meta), path("*.cram"), emit: cram, optional: true + tuple val(meta), path("*.csi"), emit: csi, optional: true + tuple val(meta), path("*.crai"), emit: crai, optional: true + tuple val(meta), path("*.log"), emit: log + tuple val(meta), path("*fastq.gz"), emit: fastq, optional: true + path "versions.yml", emit: versions when: task.ext.when == null || task.ext.when @@ -37,17 +37,20 @@ process BOWTIE2_ALIGN { if (meta.single_end) { unaligned = save_unaligned ? "--un-gz ${prefix}.unmapped.fastq.gz" : "" reads_args = "-U ${reads}" - } else { + } + else { unaligned = save_unaligned ? "--un-conc-gz ${prefix}.unmapped.fastq.gz" : "" reads_args = "-1 ${reads[0]} -2 ${reads[1]}" } def samtools_command = sort_bam ? 'sort' : 'view' def extension_pattern = /(--output-fmt|-O)+\s+(\S+)/ - def extension_matcher = (args2 =~ extension_pattern) + def extension_matcher = (args2 =~ extension_pattern) def extension = extension_matcher.getCount() > 0 ? extension_matcher[0][2].toLowerCase() : "bam" - def reference = fasta && extension=="cram" ? "--reference ${fasta}" : "" - if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" + def reference = fasta && extension == "cram" ? "--reference ${fasta}" : "" + if (!fasta && extension == "cram") { + error("Fasta reference is required for CRAM output") + } """ INDEX=`find -L ./ -name "*.rev.1.bt2" | sed "s/\\.rev.1.bt2\$//"` @@ -56,12 +59,12 @@ process BOWTIE2_ALIGN { bowtie2 \\ -x \$INDEX \\ - $reads_args \\ - --threads $task.cpus \\ - $unaligned \\ - $args \\ + ${reads_args} \\ + --threads ${task.cpus} \\ + ${unaligned} \\ + ${args} \\ 2>| >(tee ${prefix}.bowtie2.log >&2) \\ - | samtools $samtools_command $args2 --threads $task.cpus ${reference} -o ${prefix}.${extension} - + | samtools ${samtools_command} ${args2} --threads ${task.cpus} ${reference} -o ${prefix}.${extension} - if [ -f ${prefix}.unmapped.fastq.1.gz ]; then mv ${prefix}.unmapped.fastq.1.gz ${prefix}.unmapped_1.fastq.gz @@ -83,19 +86,23 @@ process BOWTIE2_ALIGN { def args2 = task.ext.args2 ?: "" def prefix = task.ext.prefix ?: "${meta.id}" def extension_pattern = /(--output-fmt|-O)+\s+(\S+)/ - def extension = (args2 ==~ extension_pattern) ? (args2 =~ extension_pattern)[0][2].toLowerCase() : "bam" + def extension = args2 ==~ extension_pattern ? (args2 =~ extension_pattern)[0][2].toLowerCase() : "bam" def create_unmapped = "" if (meta.single_end) { create_unmapped = save_unaligned ? "touch ${prefix}.unmapped.fastq.gz" : "" - } else { + } + else { create_unmapped = save_unaligned ? "touch ${prefix}.unmapped_1.fastq.gz && touch ${prefix}.unmapped_2.fastq.gz" : "" } - if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" + if (!fasta && extension == "cram") { + error("Fasta reference is required for CRAM output") + } def create_index = "" if (extension == "cram") { create_index = "touch ${prefix}.crai" - } else if (extension == "bam") { + } + else if (extension == "bam") { create_index = "touch ${prefix}.csi" } @@ -112,5 +119,4 @@ process BOWTIE2_ALIGN { pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) END_VERSIONS """ - } diff --git a/modules/nf-core/kraken2/build/environment.yml b/modules/nf-core/kraken2/build/environment.yml index 19525e8d839f..6ac6ab37b358 100644 --- a/modules/nf-core/kraken2/build/environment.yml +++ b/modules/nf-core/kraken2/build/environment.yml @@ -5,5 +5,5 @@ channels: - bioconda dependencies: - bioconda::kraken2=2.1.5 - - coreutils=9.4 + - conda-forge::coreutils=9.5 - pigz=2.8 diff --git a/modules/nf-core/kraken2/build/tests/main.nf.test b/modules/nf-core/kraken2/build/tests/main.nf.test index d3d26bcd71fe..55f5d0cb3193 100644 --- a/modules/nf-core/kraken2/build/tests/main.nf.test +++ b/modules/nf-core/kraken2/build/tests/main.nf.test @@ -10,6 +10,7 @@ nextflow_process { tag "gunzip" tag "modules" tag "modules_nfcore" + tag "arm_failure" setup { @@ -170,4 +171,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/subworkflows/nf-core/fasta_build_add_kraken2/tests/main.nf.test b/subworkflows/nf-core/fasta_build_add_kraken2/tests/main.nf.test index cfcca39878d0..83783c70f9f2 100644 --- a/subworkflows/nf-core/fasta_build_add_kraken2/tests/main.nf.test +++ b/subworkflows/nf-core/fasta_build_add_kraken2/tests/main.nf.test @@ -11,6 +11,7 @@ nextflow_workflow { tag "kraken2" tag "kraken2/add" tag "kraken2/build" + tag "arm_failure" test("metagenome - fasta - nocleanup") { @@ -154,4 +155,4 @@ nextflow_workflow { } } -} +} \ No newline at end of file diff --git a/subworkflows/nf-core/fasta_build_add_kraken2_bracken/tests/main.nf.test b/subworkflows/nf-core/fasta_build_add_kraken2_bracken/tests/main.nf.test index e036a04e5043..e250b3e5f026 100644 --- a/subworkflows/nf-core/fasta_build_add_kraken2_bracken/tests/main.nf.test +++ b/subworkflows/nf-core/fasta_build_add_kraken2_bracken/tests/main.nf.test @@ -12,6 +12,7 @@ nextflow_workflow { tag "kraken2/add" tag "kraken2/build" tag "bracken/build" + tag "arm_failure" test("metagenome - nocleanup - nobracken - fasta") { @@ -212,4 +213,4 @@ nextflow_workflow { ) } } -} +} \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test index 48988102919a..1a5ac2ed1bfb 100644 --- a/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test @@ -235,6 +235,9 @@ nextflow_workflow { } } test ("test_fastq_align_dragmap_SE"){ + + tag "arm_failure" + setup { run("DRAGMAP_HASHTABLE") { script "../../../../modules/nf-core/dragmap/hashtable/main.nf" @@ -269,6 +272,9 @@ nextflow_workflow { } } test ("test_fastq_align_dragmap_PE"){ + + tag "arm_failure" + setup { run("DRAGMAP_HASHTABLE") { script "../../../../modules/nf-core/dragmap/hashtable/main.nf" @@ -333,6 +339,7 @@ nextflow_workflow { } } test ("test_fastq_align_snapaligner_PE"){ + tag "arm_failure" setup { run("SNAPALIGNER_INDEX") { script "../../../../modules/nf-core/snapaligner/index/main.nf" @@ -362,4 +369,4 @@ nextflow_workflow { ) } } -} +} \ No newline at end of file