From fa46b889b57fdee77ce584baa18d4477d1f755fb Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 9 Jul 2026 10:49:29 +0200 Subject: [PATCH 01/18] snapzipsort: use ubam for alignment --- conf/modules.config | 4 ++-- modules/local/fgumi/snapzipsort/main.nf | 14 +++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 9da06635..28a21a4d 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -252,7 +252,7 @@ process { //// FGUMI fastq | SNAP | zipper (step 3a) withName: RAW_FGUMI_SNAPZIPSORT { ext.prefix = { "${meta.id}.fgumi" } - ext.args2 = { + ext.args = { [ "-b-", "-sm 20", @@ -264,7 +264,7 @@ process { meta.readgroup ? "-R \"@RG\\t" + meta.readgroup.findResults { rg -> rg.value?.trim() ? "${rg.key}:${rg.value}" : null }.join("\\t") + "\"" : "", ].join(" ").trim() } - ext.args4 = { + ext.args3 = { [ "--order template-coordinate", "--max-memory ${task.memory.toGiga()}G", diff --git a/modules/local/fgumi/snapzipsort/main.nf b/modules/local/fgumi/snapzipsort/main.nf index 88c34893..d6277e55 100644 --- a/modules/local/fgumi/snapzipsort/main.nf +++ b/modules/local/fgumi/snapzipsort/main.nf @@ -23,7 +23,6 @@ process FGUMI_SNAPZIPSORT { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def args3 = task.ext.args3 ?: '' - def args4 = task.ext.args4 ?: '' prefix = task.ext.prefix ?: "${meta.id}.fgumi" """ @@ -32,25 +31,22 @@ process FGUMI_SNAPZIPSORT { [ -z "\$INDEX_FILE" ] && echo "Snap index files not found" 1>&2 && exit 1 INDEX=\$(dirname "\$INDEX_FILE") - fgumi fastq \\ - --input ${unmapped_bam} \\ - ${args} \\ - | snap-aligner paired \\ + snap-aligner paired \\ \$INDEX \\ - -pairedInterleavedFastq - \\ + -bam - \\ -t ${task.cpus} \\ -o -bam - \\ - ${args2} \\ + ${args} \\ | fgumi zipper \\ --unmapped ${unmapped_bam} \\ --reference ${fasta} \\ --threads ${task.cpus} \\ - ${args3} \\ + ${args2} \\ | fgumi sort \\ --input - \\ --output ${prefix}.bam \\ --threads ${task.cpus} \\ - ${args4} + ${args3} """ stub: From 25a1340606b2b4e07b9491439b1fdfec73b505b0 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 9 Jul 2026 10:56:15 +0200 Subject: [PATCH 02/18] Update modules/local/fgumi/snapzipsort/main.nf Co-authored-by: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> --- modules/local/fgumi/snapzipsort/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/fgumi/snapzipsort/main.nf b/modules/local/fgumi/snapzipsort/main.nf index d6277e55..eb3f36e6 100644 --- a/modules/local/fgumi/snapzipsort/main.nf +++ b/modules/local/fgumi/snapzipsort/main.nf @@ -33,7 +33,7 @@ process FGUMI_SNAPZIPSORT { snap-aligner paired \\ \$INDEX \\ - -bam - \\ + -bam ${unmapped_bam} \\ -t ${task.cpus} \\ -o -bam - \\ ${args} \\ From 74f35f76df58f82df1b13279f267f6679e3ec3e8 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:28:39 +0200 Subject: [PATCH 03/18] fix sorting --- conf/modules.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/modules.config b/conf/modules.config index 28a21a4d..5ab72c9b 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -268,6 +268,7 @@ process { [ "--order template-coordinate", "--max-memory ${task.memory.toGiga()}G", + "--key-types mi" ].join(" ").trim() } } From 1bc575a1329f480db0fdb367da83b8f55acb6c5b Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:43:46 +0200 Subject: [PATCH 04/18] fix test --- tests/modules/local/fgumi/snapzipsort/main.nf.test | 1 + tests/modules/local/fgumi/snapzipsort/nextflow.config | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/modules/local/fgumi/snapzipsort/nextflow.config diff --git a/tests/modules/local/fgumi/snapzipsort/main.nf.test b/tests/modules/local/fgumi/snapzipsort/main.nf.test index 5c2c404b..f9aa8784 100644 --- a/tests/modules/local/fgumi/snapzipsort/main.nf.test +++ b/tests/modules/local/fgumi/snapzipsort/main.nf.test @@ -3,6 +3,7 @@ nextflow_process { name "Test Process FGUMI_SNAPZIPSORT" script "modules/local/fgumi/snapzipsort/main.nf" process "FGUMI_SNAPZIPSORT" + config "./nextflow.config" tag "modules" tag "modules/local" diff --git a/tests/modules/local/fgumi/snapzipsort/nextflow.config b/tests/modules/local/fgumi/snapzipsort/nextflow.config new file mode 100644 index 00000000..ec000128 --- /dev/null +++ b/tests/modules/local/fgumi/snapzipsort/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: 'FGUMI_SNAPZIPSORT' { + ext.args = "--order template-coordinate --max-memory ${task.memory.toGiga()}G --key-types mi" + } +} From 436d2651a962fb04926554b9789161145ba8170f Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:53:50 +0200 Subject: [PATCH 05/18] fix test --- tests/modules/local/fgumi/snapzipsort/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/local/fgumi/snapzipsort/nextflow.config b/tests/modules/local/fgumi/snapzipsort/nextflow.config index ec000128..7d4872e5 100644 --- a/tests/modules/local/fgumi/snapzipsort/nextflow.config +++ b/tests/modules/local/fgumi/snapzipsort/nextflow.config @@ -1,5 +1,5 @@ process { withName: 'FGUMI_SNAPZIPSORT' { - ext.args = "--order template-coordinate --max-memory ${task.memory.toGiga()}G --key-types mi" + ext.args = "--order template-coordinate --key-types mi" } } From c6d9e372ded6877897477e1f58576db7019cb9a0 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:55:10 +0200 Subject: [PATCH 06/18] fix subwf test --- .../subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test | 1 + .../local/fastq_umiconsensus_fgumi/nextflow.config | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/subworkflows/local/fastq_umiconsensus_fgumi/nextflow.config diff --git a/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test b/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test index b1b4d573..63dbe8b3 100644 --- a/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test +++ b/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test @@ -3,6 +3,7 @@ nextflow_workflow { name "Test Workflow FASTQ_UMICONSENSUS_FGUMI" script "subworkflows/local/fastq_umiconsensus_fgumi/main.nf" workflow "FASTQ_UMICONSENSUS_FGUMI" + config "./nextflow.config" tag "subworkflows" tag "subworkflows/local" diff --git a/tests/subworkflows/local/fastq_umiconsensus_fgumi/nextflow.config b/tests/subworkflows/local/fastq_umiconsensus_fgumi/nextflow.config new file mode 100644 index 00000000..7d4872e5 --- /dev/null +++ b/tests/subworkflows/local/fastq_umiconsensus_fgumi/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: 'FGUMI_SNAPZIPSORT' { + ext.args = "--order template-coordinate --key-types mi" + } +} From 5b731ad20ad66a46c81134259d7afd8956f03e69 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:13:06 +0200 Subject: [PATCH 07/18] actually set the test args --- tests/modules/local/fgumi/snapzipsort/nextflow.config | 2 +- .../subworkflows/local/fastq_umiconsensus_fgumi/nextflow.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/modules/local/fgumi/snapzipsort/nextflow.config b/tests/modules/local/fgumi/snapzipsort/nextflow.config index 7d4872e5..74ecf260 100644 --- a/tests/modules/local/fgumi/snapzipsort/nextflow.config +++ b/tests/modules/local/fgumi/snapzipsort/nextflow.config @@ -1,5 +1,5 @@ process { withName: 'FGUMI_SNAPZIPSORT' { - ext.args = "--order template-coordinate --key-types mi" + ext.args3 = "--order template-coordinate --key-types mi" } } diff --git a/tests/subworkflows/local/fastq_umiconsensus_fgumi/nextflow.config b/tests/subworkflows/local/fastq_umiconsensus_fgumi/nextflow.config index 7d4872e5..74ecf260 100644 --- a/tests/subworkflows/local/fastq_umiconsensus_fgumi/nextflow.config +++ b/tests/subworkflows/local/fastq_umiconsensus_fgumi/nextflow.config @@ -1,5 +1,5 @@ process { withName: 'FGUMI_SNAPZIPSORT' { - ext.args = "--order template-coordinate --key-types mi" + ext.args3 = "--order template-coordinate --key-types mi" } } From 22c5d97852663f185e88d365d353523958a8569a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jul 2026 10:42:42 +0200 Subject: [PATCH 08/18] fix snap --- tests/modules/local/fgumi/snapzipsort/main.nf.test.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/modules/local/fgumi/snapzipsort/main.nf.test.snap b/tests/modules/local/fgumi/snapzipsort/main.nf.test.snap index bea6ad48..7951f68d 100644 --- a/tests/modules/local/fgumi/snapzipsort/main.nf.test.snap +++ b/tests/modules/local/fgumi/snapzipsort/main.nf.test.snap @@ -11,7 +11,7 @@ "id": "test", "single_end": false }, - "test.fgumi.bam:md5,6bab0c75514fda4b4e7efbbae5e33cc8" + "test.fgumi.bam:md5,12bd726ae1623b7133398e231f866cd8" ] ] }, @@ -30,7 +30,7 @@ ] } ], - "timestamp": "2026-07-06T13:16:43.167620236", + "timestamp": "2026-07-15T10:27:42.862700628", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.4" From 95e68f06fc7cb8d92072e99c214979adff7ed85d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jul 2026 12:46:26 +0200 Subject: [PATCH 09/18] fix args --- conf/modules.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 5ab72c9b..688a6e70 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -293,7 +293,7 @@ process { //// FGUMI consensus alignment (step 8) withName: UMI_FGUMI_SNAPZIPSORT { ext.prefix = { "${meta.id}.fgumi" } - ext.args2 = { + ext.args = { [ "-b-", "-sm 20", @@ -305,7 +305,7 @@ process { meta.readgroup ? "-R \"@RG\\t" + meta.readgroup.findResults { rg -> rg.value?.trim() ? "${rg.key}:${rg.value}" : null }.join("\\t") + "\"" : "", ].join(" ").trim() } - ext.args4 = { + ext.args3 = { [ "--order coordinate", "--max-memory ${task.memory.toGiga()}G", From 0c09cd0ec07ed90ca424b9d1f18b7efb77437f1f Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jul 2026 12:51:14 +0200 Subject: [PATCH 10/18] fix memory issues --- conf/modules.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 688a6e70..d2f0de1b 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -267,7 +267,7 @@ process { ext.args3 = { [ "--order template-coordinate", - "--max-memory ${task.memory.toGiga()}G", + "--max-memory ${task.memory.toGiga() / task.cpus}G", "--key-types mi" ].join(" ").trim() } @@ -308,7 +308,7 @@ process { ext.args3 = { [ "--order coordinate", - "--max-memory ${task.memory.toGiga()}G", + "--max-memory ${task.memory.toGiga() / task.cpus}G", "--write-index", ].join(" ").trim() } From eb1afcf2e6282117fcad6f922bd3541823679638 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jul 2026 13:20:55 +0200 Subject: [PATCH 11/18] use -samNoSQ for snap input --- modules/local/fgumi/snapzipsort/main.nf | 2 +- .../local/fgumi/snapzipsort/main.nf.test.snap | 4 ++-- .../fastq_umiconsensus_fgumi/main.nf.test.snap | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/local/fgumi/snapzipsort/main.nf b/modules/local/fgumi/snapzipsort/main.nf index eb3f36e6..4ffd9d2d 100644 --- a/modules/local/fgumi/snapzipsort/main.nf +++ b/modules/local/fgumi/snapzipsort/main.nf @@ -33,7 +33,7 @@ process FGUMI_SNAPZIPSORT { snap-aligner paired \\ \$INDEX \\ - -bam ${unmapped_bam} \\ + -samNoSQ ${unmapped_bam} \\ -t ${task.cpus} \\ -o -bam - \\ ${args} \\ diff --git a/tests/modules/local/fgumi/snapzipsort/main.nf.test.snap b/tests/modules/local/fgumi/snapzipsort/main.nf.test.snap index 7951f68d..f3187dd5 100644 --- a/tests/modules/local/fgumi/snapzipsort/main.nf.test.snap +++ b/tests/modules/local/fgumi/snapzipsort/main.nf.test.snap @@ -11,7 +11,7 @@ "id": "test", "single_end": false }, - "test.fgumi.bam:md5,12bd726ae1623b7133398e231f866cd8" + "test.fgumi.bam:md5,9bfbd21ebea8ca63427cec8dd13e6063" ] ] }, @@ -30,7 +30,7 @@ ] } ], - "timestamp": "2026-07-15T10:27:42.862700628", + "timestamp": "2026-07-15T13:19:00.257043127", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.4" diff --git a/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test.snap b/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test.snap index 68e5d648..f6360e82 100644 --- a/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test.snap +++ b/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test.snap @@ -22,8 +22,8 @@ } } }, - "test.fgumi.bam:md5,d8847959ea89503e93669d3d5526813c", - "test.fgumi.bam.bai:md5,6b054c31ca1e0c5cdf99d6fdc6229654" + "test.fgumi.bam:md5,271e203740c8d65a49458ed13cb8be7d", + "test.fgumi.bam.bai:md5,0d76977b2e36046cc176112776c5fa4e" ] ], "consensus_metrics": [ @@ -46,7 +46,7 @@ } } }, - "test.fgumi.simplex.stats.txt:md5,c259ad6db7c02de9e0f1b9f95a3d6ab0" + "test.fgumi.simplex.stats.txt:md5,8729c0587ff37d0e71b466c694bc62e4" ] ], "family_size_histogram": [ @@ -69,7 +69,7 @@ } } }, - "test.fgumi.group.family_size_histogram.txt:md5,57742826c7e781f773840f1c977f5f0b" + "test.fgumi.group.family_size_histogram.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "filtering_metrics": [ @@ -92,7 +92,7 @@ } } }, - "test.fgumi.filter.stats.txt:md5,01b1b32c9151552a623eef933b8d1f96" + "test.fgumi.filter.stats.txt:md5,88b8f839f5e05e7944504f526949763c" ] ], "grouping_metrics": [ @@ -115,12 +115,12 @@ } } }, - "test.fgumi.group.grouping_metrics.txt:md5,deb6be58b11022b67145fe4e2c7ca299" + "test.fgumi.group.grouping_metrics.txt:md5,b177c7de45cf25495c61f7bced9d8416" ] ] } ], - "timestamp": "2026-07-06T13:29:57.554096841", + "timestamp": "2026-07-15T13:16:45.581965003", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.4" From 72a8b83cc14a713c0c6d4fd7643d21a7b817ce93 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jul 2026 14:08:09 +0200 Subject: [PATCH 12/18] enable deadlock recovery --- conf/modules.config | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index d2f0de1b..b32bee46 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -276,7 +276,12 @@ process { //// FGUMI group (step 4) withName: '.*FGUMI_GROUP' { ext.prefix = { "${meta.id}.fgumi.group" } - ext.args = { "--edits ${meta.fgumi_group_edits != null ? meta.fgumi_group_edits : 1}" } + ext.args = { + [ + "--edits ${meta.fgumi_group_edits != null ? meta.fgumi_group_edits : 1}" , + "--deadlock-recover" // Prevents deadlocks from occurring by temporarily bumping memory limits + ].join(" ").trim() + } } //// FGUMI simplex (step 5) From 29542d4240c1c1747d87905d5a451cdfb3572ea0 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jul 2026 14:39:33 +0200 Subject: [PATCH 13/18] remove deadlock recovery + fix memory assignment in fgumi --- conf/modules.config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index b32bee46..f06ee0a5 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -279,7 +279,7 @@ process { ext.args = { [ "--edits ${meta.fgumi_group_edits != null ? meta.fgumi_group_edits : 1}" , - "--deadlock-recover" // Prevents deadlocks from occurring by temporarily bumping memory limits + "--max-memory ${task.memory.toGiga() / task.cpus}G" ].join(" ").trim() } } @@ -287,12 +287,13 @@ process { //// FGUMI simplex (step 5) withName: '.*FGUMI_SIMPLEX' { ext.prefix = { "${meta.id}.fgumi.simplex" } - ext.args = { "--max-memory ${task.memory.toGiga()}G" } + ext.args = { "--max-memory ${task.memory.toGiga() / task.cpus}G" } } //// FGUMI filter + coordinate sort/index (step 7) withName: '.*FGUMI_FILTER' { ext.prefix = { "${meta.id}.fgumi.filter" } + ext.args = { "--max-memory ${task.memory.toGiga() / task.cpus}G" } } //// FGUMI consensus alignment (step 8) From ef63dbaaba0e5ad644e931beb81db1e932a1db8a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jul 2026 16:00:51 +0200 Subject: [PATCH 14/18] fix joining --- subworkflows/local/fastq_umiconsensus_fgumi/main.nf | 5 ++--- .../local/fastq_umiconsensus_fgumi/main.nf.test.snap | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/subworkflows/local/fastq_umiconsensus_fgumi/main.nf b/subworkflows/local/fastq_umiconsensus_fgumi/main.nf index e2416f9b..3b77450d 100644 --- a/subworkflows/local/fastq_umiconsensus_fgumi/main.nf +++ b/subworkflows/local/fastq_umiconsensus_fgumi/main.nf @@ -84,7 +84,7 @@ workflow FASTQ_UMICONSENSUS_FGUMI { // Step 7: filter consensus reads, then coordinate-sort/index for downstream CRAM conversion. FGUMI_FILTER( - FGUMI_SIMPLEX.out.bam.join(ch_meta_fastqs).map { meta, simplex_bams, _fastqs -> + FGUMI_SIMPLEX.out.bam.map { meta, simplex_bams -> [meta, simplex_bams, getGenomeAttribute(meta.genome_data, 'fasta')] }, '1,1,1', @@ -93,8 +93,7 @@ workflow FASTQ_UMICONSENSUS_FGUMI { UMI_FGUMI_SNAPZIPSORT( FGUMI_FILTER.out.bam - .join(ch_meta_fastqs) - .map { meta, filtered_bams, _fastqs -> + .map { meta, filtered_bams -> [ meta, filtered_bams, diff --git a/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test.snap b/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test.snap index f6360e82..27975a12 100644 --- a/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test.snap +++ b/tests/subworkflows/local/fastq_umiconsensus_fgumi/main.nf.test.snap @@ -22,7 +22,7 @@ } } }, - "test.fgumi.bam:md5,271e203740c8d65a49458ed13cb8be7d", + "test.fgumi.bam:md5,c4a33382a1fb8a6361d23928359d672f", "test.fgumi.bam.bai:md5,0d76977b2e36046cc176112776c5fa4e" ] ], @@ -120,7 +120,7 @@ ] } ], - "timestamp": "2026-07-15T13:16:45.581965003", + "timestamp": "2026-07-15T15:59:40.968881197", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.4" From f08b369c62530b4ee72d258ef4672d523f66dd83 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jul 2026 16:18:14 +0200 Subject: [PATCH 15/18] fix snap + run prek --- conf/modules.config | 2 +- .../fastq_to_aligned_cram/main.nf.test.snap | 54 +++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index f06ee0a5..1dc32dd5 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -276,7 +276,7 @@ process { //// FGUMI group (step 4) withName: '.*FGUMI_GROUP' { ext.prefix = { "${meta.id}.fgumi.group" } - ext.args = { + ext.args = { [ "--edits ${meta.fgumi_group_edits != null ? meta.fgumi_group_edits : 1}" , "--max-memory ${task.memory.toGiga() / task.cpus}G" diff --git a/tests/subworkflows/local/fastq_to_aligned_cram/main.nf.test.snap b/tests/subworkflows/local/fastq_to_aligned_cram/main.nf.test.snap index fa0f4ff7..844972d2 100644 --- a/tests/subworkflows/local/fastq_to_aligned_cram/main.nf.test.snap +++ b/tests/subworkflows/local/fastq_to_aligned_cram/main.nf.test.snap @@ -3,7 +3,7 @@ "content": [ { "align_reports": [ - + ], "cram_crai": [ [ @@ -26,16 +26,16 @@ ] ], "family_size_histogram": [ - + ], "rna_junctions": [ - + ], "rna_splice_junctions": [ - + ], "sormadup_metrics": [ - + ] } ], @@ -49,7 +49,7 @@ "content": [ { "align_reports": [ - + ], "cram_crai": [ [ @@ -72,13 +72,13 @@ ] ], "family_size_histogram": [ - + ], "rna_junctions": [ - + ], "rna_splice_junctions": [ - + ], "sormadup_metrics": [ [ @@ -111,7 +111,7 @@ "content": [ { "align_reports": [ - + ], "cram_crai": [ [ @@ -134,13 +134,13 @@ ] ], "family_size_histogram": [ - + ], "rna_junctions": [ - + ], "rna_splice_junctions": [ - + ], "sormadup_metrics": [ [ @@ -173,7 +173,7 @@ "content": [ { "align_reports": [ - + ], "cram_crai": [ [ @@ -197,7 +197,7 @@ ] ], "family_size_histogram": [ - + ], "rna_junctions": [ [ @@ -271,7 +271,7 @@ "content": [ { "align_reports": [ - + ], "cram_crai": [ [ @@ -294,16 +294,16 @@ ] ], "family_size_histogram": [ - + ], "rna_junctions": [ - + ], "rna_splice_junctions": [ - + ], "sormadup_metrics": [ - + ] } ], @@ -317,7 +317,7 @@ "content": [ { "align_reports": [ - + ], "cram_crai": [ [ @@ -365,24 +365,24 @@ } } }, - "test.fgumi.group.family_size_histogram.txt:md5,57742826c7e781f773840f1c977f5f0b" + "test.fgumi.group.family_size_histogram.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "rna_junctions": [ - + ], "rna_splice_junctions": [ - + ], "sormadup_metrics": [ - + ] } ], - "timestamp": "2026-07-06T13:26:52.23745498", + "timestamp": "2026-07-15T16:14:27.2930013", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.4" } } -} +} \ No newline at end of file From 33b15021c643fe7e39b4a4989ec983fd267ba380 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 16 Jul 2026 11:27:22 +0200 Subject: [PATCH 16/18] remove samNoSQ and let snap decide for itself --- modules/local/fgumi/snapzipsort/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/fgumi/snapzipsort/main.nf b/modules/local/fgumi/snapzipsort/main.nf index 4ffd9d2d..d9a6ea7d 100644 --- a/modules/local/fgumi/snapzipsort/main.nf +++ b/modules/local/fgumi/snapzipsort/main.nf @@ -33,7 +33,7 @@ process FGUMI_SNAPZIPSORT { snap-aligner paired \\ \$INDEX \\ - -samNoSQ ${unmapped_bam} \\ + ${unmapped_bam} \\ -t ${task.cpus} \\ -o -bam - \\ ${args} \\ From 84cca63302499e378c48176abfe5e0220bbd53a6 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 16 Jul 2026 11:49:37 +0200 Subject: [PATCH 17/18] add queryname sort before mapping --- conf/modules.config | 12 +++ modules.json | 5 ++ modules/nf-core/fgumi/sort/environment.yml | 7 ++ modules/nf-core/fgumi/sort/main.nf | 45 ++++++++++ modules/nf-core/fgumi/sort/meta.yml | 82 +++++++++++++++++ modules/nf-core/fgumi/sort/tests/main.nf.test | 89 +++++++++++++++++++ .../fgumi/sort/tests/main.nf.test.snap | 87 ++++++++++++++++++ .../nf-core/fgumi/sort/tests/nextflow.config | 5 ++ .../local/fastq_umiconsensus_fgumi/main.nf | 7 +- 9 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 modules/nf-core/fgumi/sort/environment.yml create mode 100644 modules/nf-core/fgumi/sort/main.nf create mode 100644 modules/nf-core/fgumi/sort/meta.yml create mode 100644 modules/nf-core/fgumi/sort/tests/main.nf.test create mode 100644 modules/nf-core/fgumi/sort/tests/main.nf.test.snap create mode 100644 modules/nf-core/fgumi/sort/tests/nextflow.config diff --git a/conf/modules.config b/conf/modules.config index 1dc32dd5..2286ab89 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -249,6 +249,18 @@ process { } } + withName: '.*FGUMI_SORT' { + ext.prefix = { "${meta.id}.fgumi.unmapped.sorted" } + ext.args = { + [ + "--order queryname", + "--compression-level 0", + "--max-memory ${task.memory.toGiga() / task.cpus}G", + "--tmp-dir .", + ].join(" ").trim() + } + } + //// FGUMI fastq | SNAP | zipper (step 3a) withName: RAW_FGUMI_SNAPZIPSORT { ext.prefix = { "${meta.id}.fgumi" } diff --git a/modules.json b/modules.json index ccf88121..5bbcd04d 100644 --- a/modules.json +++ b/modules.json @@ -77,6 +77,11 @@ "installed_by": ["modules"], "patch": "modules/nf-core/fgumi/simplex/fgumi-simplex.diff" }, + "fgumi/sort": { + "branch": "master", + "git_sha": "c3e04783b2ac99ffc011745ec092614a7bef86ca", + "installed_by": ["modules"] + }, "gnu/sort": { "branch": "master", "git_sha": "6d46786420b4d7bc88eba026eb389c0c5535d120", diff --git a/modules/nf-core/fgumi/sort/environment.yml b/modules/nf-core/fgumi/sort/environment.yml new file mode 100644 index 00000000..68aafa2b --- /dev/null +++ b/modules/nf-core/fgumi/sort/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "bioconda::fgumi=0.4.0" diff --git a/modules/nf-core/fgumi/sort/main.nf b/modules/nf-core/fgumi/sort/main.nf new file mode 100644 index 00000000..a3262e5d --- /dev/null +++ b/modules/nf-core/fgumi/sort/main.nf @@ -0,0 +1,45 @@ +process FGUMI_SORT { + tag "${meta.id}" + label 'process_medium' + + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/4a/4a62b457c53300603da026225f95b4db04d1c9f8ba7f734787818fc105d51323/data' + : 'community.wave.seqera.io/library/fgumi:0.4.0--1fb5dc6de05ce63b'}" + + input: + tuple val(meta), path(bam) + + output: + tuple val(meta), path("*.bam"), emit: bam + tuple val(meta), path("*.{csi,bai}"), emit: index, optional: true + tuple val("${task.process}"), val('fgumi'), eval('fgumi --version | sed "s/^fgumi //"'), topic: versions, emit: versions_fgumi + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}_sorted" + + if ("${bam}" == "${prefix}.bam") { + error("Input and output names are the same, use \"task.ext.prefix\" to disambiguate!") + } + + """ + fgumi sort \\ + --input ${bam} \\ + --output ${prefix}.bam \\ + --threads ${task.cpus} \\ + ${args} + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}_sorted" + if ("${bam}" == "${prefix}.bam") { + error("Input and output names are the same, use \"task.ext.prefix\" to disambiguate!") + } + """ + touch ${prefix}.bam + """ +} diff --git a/modules/nf-core/fgumi/sort/meta.yml b/modules/nf-core/fgumi/sort/meta.yml new file mode 100644 index 00000000..1dc25228 --- /dev/null +++ b/modules/nf-core/fgumi/sort/meta.yml @@ -0,0 +1,82 @@ +name: "fgumi_sort" +description: | + Sorts a SAM or BAM file. Several sort orders are available, including coordinate, + queryname, and template-coordinate. This is a high-performance replacement for fgbio SortBam. +keywords: + - sort + - bam + - sam +tools: + - "fgumi": + description: "High-performance tools for working with UMI-tagged sequencing data." + homepage: "https://github.com/fulcrumgenomics/fgumi" + documentation: "https://docs.rs/fgumi" + tool_dev_url: "https://github.com/fulcrumgenomics/fgumi" + licence: + - "MIT" + identifier: "" +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - bam: + type: file + description: | + The input SAM or BAM file to be sorted. + pattern: "*.{bam,sam}" + ontologies: + - edam: "http://edamontology.org/format_2572" +output: + bam: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - "*.bam": + type: file + description: | + Sorted output BAM file. + pattern: "*.bam" + ontologies: + - edam: "http://edamontology.org/format_2572" + index: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - "*.{csi,bai}": + type: file + description: | + Index file if the bam file is coordinate sorted. + pattern: "*.{csi,bai}" + ontologies: + - edam: "http://edamontology.org/format_3327" + versions_fgumi: + - - ${task.process}: + type: string + description: The process the versions were collected from + - fgumi: + type: string + description: The tool name + - 'fgumi --version | sed "s/^fgumi //"': + type: eval + description: The expression to obtain the version of the tool +topics: + versions: + - - ${task.process}: + type: string + description: The process the versions were collected from + - fgumi: + type: string + description: The tool name + - 'fgumi --version | sed "s/^fgumi //"': + type: eval + description: The expression to obtain the version of the tool +authors: + - "@sppearce" +maintainers: + - "@sppearce" diff --git a/modules/nf-core/fgumi/sort/tests/main.nf.test b/modules/nf-core/fgumi/sort/tests/main.nf.test new file mode 100644 index 00000000..fc549bf5 --- /dev/null +++ b/modules/nf-core/fgumi/sort/tests/main.nf.test @@ -0,0 +1,89 @@ +nextflow_process { + + name "Test Process FGUMI_SORT" + script "../main.nf" + process "FGUMI_SORT" + + tag "modules" + tag "modules_nfcore" + tag "fgumi" + tag "fgumi/sort" + + test("sarscov2 - bam") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + ] + """ + } + } + + then { + assert process.success + assertAll( + { assert snapshot( + bam(process.out.bam.get(0).get(1)).getReadsMD5(), + process.out.findAll { key, val -> key.startsWith('versions') } + ).match() } + ) + } + + } + + test("sarscov2 - bam - template-coordinate") { + + when { + params { + module_args = '--order template-coordinate' + } + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + ] + """ + } + } + + then { + assert process.success + assertAll( + { assert snapshot( + bam(process.out.bam.get(0).get(1)).getReadsMD5(), + process.out.findAll { key, val -> key.startsWith('versions') } + ).match() } + ) + } + + } + + test("sarscov2 - bam - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + ] + """ + } + } + + then { + assert process.success + assertAll( + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/fgumi/sort/tests/main.nf.test.snap b/modules/nf-core/fgumi/sort/tests/main.nf.test.snap new file mode 100644 index 00000000..ad6dfc3b --- /dev/null +++ b/modules/nf-core/fgumi/sort/tests/main.nf.test.snap @@ -0,0 +1,87 @@ +{ + "sarscov2 - bam - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test_sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + + ], + "2": [ + [ + "FGUMI_SORT", + "fgumi", + "0.4.0" + ] + ], + "bam": [ + [ + { + "id": "test" + }, + "test_sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "index": [ + + ], + "versions_fgumi": [ + [ + "FGUMI_SORT", + "fgumi", + "0.4.0" + ] + ] + } + ], + "timestamp": "2026-06-26T13:03:56.414869", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.10.4" + } + }, + "sarscov2 - bam": { + "content": [ + "461d8083b03a321eb1902ad544fd7d2f", + { + "versions_fgumi": [ + [ + "FGUMI_SORT", + "fgumi", + "0.4.0" + ] + ] + } + ], + "timestamp": "2026-06-26T13:03:43.67993", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.10.4" + } + }, + "sarscov2 - bam - template-coordinate": { + "content": [ + "461d8083b03a321eb1902ad544fd7d2f", + { + "versions_fgumi": [ + [ + "FGUMI_SORT", + "fgumi", + "0.4.0" + ] + ] + } + ], + "timestamp": "2026-06-26T13:03:47.645547", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.10.4" + } + } +} \ No newline at end of file diff --git a/modules/nf-core/fgumi/sort/tests/nextflow.config b/modules/nf-core/fgumi/sort/tests/nextflow.config new file mode 100644 index 00000000..4ad67e9b --- /dev/null +++ b/modules/nf-core/fgumi/sort/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: FGUMI_SORT { + ext.args = { params.module_args } + } +} diff --git a/subworkflows/local/fastq_umiconsensus_fgumi/main.nf b/subworkflows/local/fastq_umiconsensus_fgumi/main.nf index 3b77450d..2353e259 100644 --- a/subworkflows/local/fastq_umiconsensus_fgumi/main.nf +++ b/subworkflows/local/fastq_umiconsensus_fgumi/main.nf @@ -2,6 +2,7 @@ // MODULES include { FGUMI_EXTRACT } from "../../../modules/nf-core/fgumi/extract/main.nf" +include { FGUMI_SORT } from "../../../modules/nf-core/fgumi/sort/main.nf" include { FGUMI_FILTER } from "../../../modules/nf-core/fgumi/filter/main.nf" include { FGUMI_GROUP } from "../../../modules/nf-core/fgumi/group/main.nf" include { FGUMI_MERGE } from "../../../modules/nf-core/fgumi/merge/main.nf" @@ -25,9 +26,13 @@ workflow FASTQ_UMICONSENSUS_FGUMI { .map { meta, fastqs -> [meta, fastqs, (meta.readgroup?.LB ?: meta.library ?: meta.id)] } ) + FGUMI_SORT( + FGUMI_EXTRACT.out.bam + ) + // Step 3: align with SNAP, zipper tags back, then template-coordinate sort. RAW_FGUMI_SNAPZIPSORT( - FGUMI_EXTRACT.out.bam + FGUMI_SORT.out.bam .join(ch_meta_fastqs) .map { meta, ubams, _fastqs -> [ From f2f3a183d23e1a3b860777248e7ae72f2879eae9 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 16 Jul 2026 13:47:44 +0200 Subject: [PATCH 18/18] add sorting directly to mapping output --- conf/modules.config | 26 ++++++++++++++++++++++--- modules/local/fgumi/snapzipsort/main.nf | 14 ++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 2286ab89..5cbc5b31 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -257,6 +257,7 @@ process { "--compression-level 0", "--max-memory ${task.memory.toGiga() / task.cpus}G", "--tmp-dir .", + "--key-types mi" ].join(" ").trim() } } @@ -276,13 +277,23 @@ process { meta.readgroup ? "-R \"@RG\\t" + meta.readgroup.findResults { rg -> rg.value?.trim() ? "${rg.key}:${rg.value}" : null }.join("\\t") + "\"" : "", ].join(" ").trim() } - ext.args3 = { + ext.args2 = { [ - "--order template-coordinate", + "--order queryname", + "--compression-level 0", "--max-memory ${task.memory.toGiga() / task.cpus}G", + "--tmp-dir .", "--key-types mi" ].join(" ").trim() } + ext.args4 = { + [ + "--order template-coordinate", + "--max-memory ${task.memory.toGiga() / task.cpus}G", + "--key-types mi", + "--tmp-dir .", + ].join(" ").trim() + } } //// FGUMI group (step 4) @@ -323,7 +334,16 @@ process { meta.readgroup ? "-R \"@RG\\t" + meta.readgroup.findResults { rg -> rg.value?.trim() ? "${rg.key}:${rg.value}" : null }.join("\\t") + "\"" : "", ].join(" ").trim() } - ext.args3 = { + ext.args2 = { + [ + "--order queryname", + "--compression-level 0", + "--max-memory ${task.memory.toGiga() / task.cpus}G", + "--tmp-dir .", + "--key-types mi" + ].join(" ").trim() + } + ext.args4 = { [ "--order coordinate", "--max-memory ${task.memory.toGiga() / task.cpus}G", diff --git a/modules/local/fgumi/snapzipsort/main.nf b/modules/local/fgumi/snapzipsort/main.nf index d9a6ea7d..9139df7a 100644 --- a/modules/local/fgumi/snapzipsort/main.nf +++ b/modules/local/fgumi/snapzipsort/main.nf @@ -23,6 +23,7 @@ process FGUMI_SNAPZIPSORT { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def args3 = task.ext.args3 ?: '' + def args4 = task.ext.args4 ?: '' prefix = task.ext.prefix ?: "${meta.id}.fgumi" """ @@ -37,16 +38,23 @@ process FGUMI_SNAPZIPSORT { -t ${task.cpus} \\ -o -bam - \\ ${args} \\ - | fgumi zipper \\ + | fgumi sort \\ + --input - \\ + --output ${prefix}.intermediate.bam \\ + --threads ${task.cpus} \\ + ${args2} + + fgumi zipper \\ + --input ${prefix}.intermediate.bam \\ --unmapped ${unmapped_bam} \\ --reference ${fasta} \\ --threads ${task.cpus} \\ - ${args2} \\ + ${args3} \\ | fgumi sort \\ --input - \\ --output ${prefix}.bam \\ --threads ${task.cpus} \\ - ${args3} + ${args4} """ stub: